diff --git a/Makefile b/Makefile index 3ab7e257a..b68c61130 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ install-3.9: ## Install and use Python 3.9 for generating test data .PHONY: ci ci: ## Run all the CI checks ci: CI=1 -ci: lint test check-codegen +ci: lint test test-type-hints check-codegen .PHONY: codegen codegen: ## Generate all the code (models, services, examples, and init files) @@ -44,7 +44,11 @@ lint: ## Run a `lint` process on Hera and report problems .PHONY: test test: ## Run tests for Hera - @poetry run python -m pytest --cov-report=term-missing -m "not on_cluster" + @poetry run python -m pytest --cov-report=term-missing -m "not on_cluster" -k "not typehints" + +.PHONY: test-type-hints +test-type-hints: ## Run type hint tests for Hera + @poetry run python -m pytest -k "typehints" .PHONY: workflows-models workflows-models: ## Generate the Workflows models portion of Argo Workflows @@ -129,7 +133,8 @@ regenerate-example: install regenerate-test-data: ## Regenerates the test data from upstream examples and runs tests, report missing examples regenerate-test-data: install-3.9 find examples -name "*.yaml" -type f -delete - HERA_REGENERATE=1 make test examples + HERA_REGENERATE=1 poetry run python -m pytest -k test_examples + make examples @poetry run python -m pytest -k test_for_missing_examples --runxfail .PHONY: install-k3d diff --git a/docs/examples/workflows-examples.md b/docs/examples/workflows-examples.md index cb2b561d0..186aff102 100644 --- a/docs/examples/workflows-examples.md +++ b/docs/examples/workflows-examples.md @@ -32,6 +32,8 @@ Explore the examples through the side bar! | [conditionals-complex](https://github.com/argoproj/argo-workflows/blob/main/examples/conditionals-complex.yaml) | | [configmaps/simple-parameters-configmap](https://github.com/argoproj/argo-workflows/blob/main/examples/configmaps/simple-parameters-configmap.yaml) | | [cron-backfill](https://github.com/argoproj/argo-workflows/blob/main/examples/cron-backfill.yaml) | +| [cron-when](https://github.com/argoproj/argo-workflows/blob/main/examples/cron-when.yaml) | +| [cron-workflow-multiple-schedules](https://github.com/argoproj/argo-workflows/blob/main/examples/cron-workflow-multiple-schedules.yaml) | | [daemon-step](https://github.com/argoproj/argo-workflows/blob/main/examples/daemon-step.yaml) | | [daemoned-stateful-set-with-service](https://github.com/argoproj/argo-workflows/blob/main/examples/daemoned-stateful-set-with-service.yaml) | | [dag-coinflip](https://github.com/argoproj/argo-workflows/blob/main/examples/dag-coinflip.yaml) | @@ -105,7 +107,11 @@ Explore the examples through the side bar! | [secrets](https://github.com/argoproj/argo-workflows/blob/main/examples/secrets.yaml) | | [status-reference](https://github.com/argoproj/argo-workflows/blob/main/examples/status-reference.yaml) | | [step-level-timeout](https://github.com/argoproj/argo-workflows/blob/main/examples/step-level-timeout.yaml) | +| [synchronization-mutex-tmpl-level](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-mutex-tmpl-level.yaml) | | [synchronization-mutex-wf-level](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-mutex-wf-level.yaml) | +| [synchronization-mutex-wf-level-legacy](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-mutex-wf-level-legacy.yaml) | +| [synchronization-tmpl-level](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-tmpl-level.yaml) | +| [synchronization-wf-level](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-wf-level.yaml) | | [template-defaults](https://github.com/argoproj/argo-workflows/blob/main/examples/template-defaults.yaml) | | [testvolume](https://github.com/argoproj/argo-workflows/blob/main/examples/testvolume.yaml) | | [timeouts-step](https://github.com/argoproj/argo-workflows/blob/main/examples/timeouts-step.yaml) | diff --git a/docs/examples/workflows/upstream/synchronization_mutex_tmpl_level.md b/docs/examples/workflows/upstream/synchronization_mutex_tmpl_level_legacy.md similarity index 97% rename from docs/examples/workflows/upstream/synchronization_mutex_tmpl_level.md rename to docs/examples/workflows/upstream/synchronization_mutex_tmpl_level_legacy.md index 55e68b003..355b59076 100644 --- a/docs/examples/workflows/upstream/synchronization_mutex_tmpl_level.md +++ b/docs/examples/workflows/upstream/synchronization_mutex_tmpl_level_legacy.md @@ -1,9 +1,9 @@ -# Synchronization Mutex Tmpl Level +# Synchronization Mutex Tmpl Level Legacy ## Note This example is a replication of an Argo Workflow example in Hera. -The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-mutex-tmpl-level.yaml). +The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-mutex-tmpl-level-legacy.yaml). diff --git a/docs/examples/workflows/upstream/synchronization_tmpl_level.md b/docs/examples/workflows/upstream/synchronization_tmpl_level_legacy.md similarity index 97% rename from docs/examples/workflows/upstream/synchronization_tmpl_level.md rename to docs/examples/workflows/upstream/synchronization_tmpl_level_legacy.md index ef8865363..f7d013d42 100644 --- a/docs/examples/workflows/upstream/synchronization_tmpl_level.md +++ b/docs/examples/workflows/upstream/synchronization_tmpl_level_legacy.md @@ -1,9 +1,9 @@ -# Synchronization Tmpl Level +# Synchronization Tmpl Level Legacy ## Note This example is a replication of an Argo Workflow example in Hera. -The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-tmpl-level.yaml). +The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-tmpl-level-legacy.yaml). diff --git a/docs/examples/workflows/upstream/synchronization_wf_level.md b/docs/examples/workflows/upstream/synchronization_wf_level_legacy.md similarity index 92% rename from docs/examples/workflows/upstream/synchronization_wf_level.md rename to docs/examples/workflows/upstream/synchronization_wf_level_legacy.md index ba01217d8..a6e75e336 100644 --- a/docs/examples/workflows/upstream/synchronization_wf_level.md +++ b/docs/examples/workflows/upstream/synchronization_wf_level_legacy.md @@ -1,9 +1,9 @@ -# Synchronization Wf Level +# Synchronization Wf Level Legacy ## Note This example is a replication of an Argo Workflow example in Hera. -The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-wf-level.yaml). +The upstream example can be [found here](https://github.com/argoproj/argo-workflows/blob/main/examples/synchronization-wf-level-legacy.yaml). diff --git a/docs/examples/workflows/use-cases/dask.md b/docs/examples/workflows/use-cases/dask.md index fe10f0158..0048f1230 100644 --- a/docs/examples/workflows/use-cases/dask.md +++ b/docs/examples/workflows/use-cases/dask.md @@ -88,7 +88,7 @@ This example showcases how to run Dask within a Hera submitted Argo workflow. import dask.array as da from dask.distributed import Client from dask_kubernetes.operator import KubeCluster - cluster = KubeCluster(image='ghcr.io/dask/dask:latest', resources={"requests": {"memory": "2G", "cpu": "1"}, "limits": {"memory": "4G", "cpu": "1"}}, namespace=namespace, n_workers=n_workers) + cluster = KubeCluster(image='ghcr.io/dask/dask:latest', resources={'requests': {'memory': '2G', 'cpu': '1'}, 'limits': {'memory': '4G', 'cpu': '1'}}, namespace=namespace, n_workers=n_workers) client = Client(cluster) array = da.ones((1000, 1000, 1000)) print('Array mean = {array_mean}, expected = 1.0'.format(array_mean=array.mean().compute())) diff --git a/examples/workflows/upstream/cron-when.upstream.yaml b/examples/workflows/upstream/cron-when.upstream.yaml new file mode 100644 index 000000000..c7904cd5a --- /dev/null +++ b/examples/workflows/upstream/cron-when.upstream.yaml @@ -0,0 +1,16 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: sleep-when +spec: + schedule: "* * * * *" + concurrencyPolicy: "Allow" + when: "{{= cronworkflow.lastScheduledTime == nil || (now() - cronworkflow.lastScheduledTime).Seconds() > 360 }}" + workflowSpec: + entrypoint: sleep-busybox + templates: + - name: sleep-busybox + container: + image: busybox + command: [sleep] + args: ["10"] diff --git a/examples/workflows/upstream/cron-workflow-multiple-schedules.upstream.yaml b/examples/workflows/upstream/cron-workflow-multiple-schedules.upstream.yaml new file mode 100644 index 000000000..5a6b8ebac --- /dev/null +++ b/examples/workflows/upstream/cron-workflow-multiple-schedules.upstream.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: hello-world-multiple-schedules +spec: + schedules: # v3.6 and after + - "*/3 * * * *" + - "*/2 * * * *" + timezone: "America/Los_Angeles" # Default to local machine timezone + startingDeadlineSeconds: 0 + concurrencyPolicy: "Replace" # Default to "Allow" + successfulJobsHistoryLimit: 4 # Default 3 + failedJobsHistoryLimit: 4 # Default 1 + suspend: false # Set to "true" to suspend scheduling + workflowSpec: + entrypoint: whalesay + templates: + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["🕓 hello world. Scheduled on: {{workflow.scheduledTime}}"] diff --git a/examples/workflows/upstream/synchronization-mutex-tmpl-level-legacy.upstream.yaml b/examples/workflows/upstream/synchronization-mutex-tmpl-level-legacy.upstream.yaml new file mode 100644 index 000000000..9652e48a7 --- /dev/null +++ b/examples/workflows/upstream/synchronization-mutex-tmpl-level-legacy.upstream.yaml @@ -0,0 +1,45 @@ +# This example demonstrates the use of a Synchronization Mutex lock on template execution. Mutex lock limits +# only one of the template execution across the workflows in the namespace which has same Mutex lock. + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: synchronization-tmpl-level-mutex- +spec: + entrypoint: synchronization-tmpl-level-mutex-example + templates: + - name: synchronization-tmpl-level-mutex-example + steps: + - - name: synchronization-acquire-lock + template: acquire-lock + arguments: + parameters: + - name: seconds + value: "{{item}}" + withParam: '["1","2","3","4","5"]' + + - name: synchronization-acquire-lock1 + template: acquire-lock-1 + arguments: + parameters: + - name: seconds + value: "{{item}}" + withParam: '["1","2","3","4","5"]' + + - name: acquire-lock + synchronization: + mutex: + name: welcome + container: + image: alpine:latest + command: [sh, -c] + args: ["sleep 20; echo acquired lock"] + + - name: acquire-lock-1 + synchronization: + mutex: + name: test + container: + image: alpine:latest + command: [sh, -c] + args: ["sleep 50; echo acquired lock"] \ No newline at end of file diff --git a/examples/workflows/upstream/synchronization-mutex-tmpl-level.yaml b/examples/workflows/upstream/synchronization-mutex-tmpl-level-legacy.yaml similarity index 100% rename from examples/workflows/upstream/synchronization-mutex-tmpl-level.yaml rename to examples/workflows/upstream/synchronization-mutex-tmpl-level-legacy.yaml diff --git a/examples/workflows/upstream/synchronization-mutex-tmpl-level.upstream.yaml b/examples/workflows/upstream/synchronization-mutex-tmpl-level.upstream.yaml index 9652e48a7..c2368cca8 100644 --- a/examples/workflows/upstream/synchronization-mutex-tmpl-level.upstream.yaml +++ b/examples/workflows/upstream/synchronization-mutex-tmpl-level.upstream.yaml @@ -28,8 +28,8 @@ spec: - name: acquire-lock synchronization: - mutex: - name: welcome + mutexes: + - name: welcome container: image: alpine:latest command: [sh, -c] @@ -37,9 +37,9 @@ spec: - name: acquire-lock-1 synchronization: - mutex: - name: test + mutexes: + - name: test container: image: alpine:latest command: [sh, -c] - args: ["sleep 50; echo acquired lock"] \ No newline at end of file + args: ["sleep 50; echo acquired lock"] diff --git a/examples/workflows/upstream/synchronization-mutex-wf-level-legacy.upstream.yaml b/examples/workflows/upstream/synchronization-mutex-wf-level-legacy.upstream.yaml new file mode 100644 index 000000000..f13b199c6 --- /dev/null +++ b/examples/workflows/upstream/synchronization-mutex-wf-level-legacy.upstream.yaml @@ -0,0 +1,17 @@ +# This example demonstrates the use of a Synchronization Mutex lock on workflow execution. Mutex lock limits +# only one of the workflow execution in the namespace which has same Mutex lock. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: synchronization-wf-level- +spec: + entrypoint: hello-world + synchronization: + mutex: + name: test + templates: + - name: hello-world + container: + image: busybox + command: [echo] + args: ["hello world"] diff --git a/examples/workflows/upstream/synchronization-mutex-wf-level.upstream.yaml b/examples/workflows/upstream/synchronization-mutex-wf-level.upstream.yaml index f13b199c6..00494c3a8 100644 --- a/examples/workflows/upstream/synchronization-mutex-wf-level.upstream.yaml +++ b/examples/workflows/upstream/synchronization-mutex-wf-level.upstream.yaml @@ -7,8 +7,8 @@ metadata: spec: entrypoint: hello-world synchronization: - mutex: - name: test + mutexes: + - name: test templates: - name: hello-world container: diff --git a/examples/workflows/upstream/synchronization-tmpl-level-legacy.upstream.yaml b/examples/workflows/upstream/synchronization-tmpl-level-legacy.upstream.yaml new file mode 100644 index 000000000..b8c0f79c6 --- /dev/null +++ b/examples/workflows/upstream/synchronization-tmpl-level-legacy.upstream.yaml @@ -0,0 +1,37 @@ +# This example demonstrates the use of a Synchronization lock on template execution. Synchronization lock limits +# the number of concurrent template execution across the workflows in the namespace which has same Synchronization lock. +# Synchronization limit value can be configured in configmap. Eg.: +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: my-config +# data: +# template: "3" +#--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: synchronization-tmpl-level- +spec: + entrypoint: synchronization-tmpl-level-example + templates: + - name: synchronization-tmpl-level-example + steps: + - - name: synchronization-acquire-lock + template: acquire-lock + arguments: + parameters: + - name: seconds + value: "{{item}}" + withParam: '["1","2","3","4","5"]' + + - name: acquire-lock + synchronization: + semaphore: + configMapKeyRef: + name: my-config + key: template + container: + image: alpine:latest + command: [sh, -c] + args: ["sleep 10; echo acquired lock"] diff --git a/examples/workflows/upstream/synchronization-tmpl-level.yaml b/examples/workflows/upstream/synchronization-tmpl-level-legacy.yaml similarity index 100% rename from examples/workflows/upstream/synchronization-tmpl-level.yaml rename to examples/workflows/upstream/synchronization-tmpl-level-legacy.yaml diff --git a/examples/workflows/upstream/synchronization-tmpl-level.upstream.yaml b/examples/workflows/upstream/synchronization-tmpl-level.upstream.yaml index b8c0f79c6..d9eafd230 100644 --- a/examples/workflows/upstream/synchronization-tmpl-level.upstream.yaml +++ b/examples/workflows/upstream/synchronization-tmpl-level.upstream.yaml @@ -27,10 +27,10 @@ spec: - name: acquire-lock synchronization: - semaphore: - configMapKeyRef: - name: my-config - key: template + semaphores: + - configMapKeyRef: + name: my-config + key: template container: image: alpine:latest command: [sh, -c] diff --git a/examples/workflows/upstream/synchronization-wf-level-legacy.upstream.yaml b/examples/workflows/upstream/synchronization-wf-level-legacy.upstream.yaml new file mode 100644 index 000000000..b7ed90a09 --- /dev/null +++ b/examples/workflows/upstream/synchronization-wf-level-legacy.upstream.yaml @@ -0,0 +1,28 @@ +# This example demonstrates the use of a Synchronization lock on workflow execution. Synchronization lock limits +# the number of concurrent workflow execution in the namespace which has same Synchronization lock. Synchronization +# limit value can be configured in configmap. +# Eg.: +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: my-config +# data: +# workflow: "3" +#--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: synchronization-wf-level- +spec: + entrypoint: hello-world + synchronization: + semaphore: + configMapKeyRef: + name: my-config + key: workflow + templates: + - name: hello-world + container: + image: busybox + command: [echo] + args: ["hello world"] diff --git a/examples/workflows/upstream/synchronization-wf-level.yaml b/examples/workflows/upstream/synchronization-wf-level-legacy.yaml similarity index 100% rename from examples/workflows/upstream/synchronization-wf-level.yaml rename to examples/workflows/upstream/synchronization-wf-level-legacy.yaml diff --git a/examples/workflows/upstream/synchronization-wf-level.upstream.yaml b/examples/workflows/upstream/synchronization-wf-level.upstream.yaml index b7ed90a09..33a8e11f7 100644 --- a/examples/workflows/upstream/synchronization-wf-level.upstream.yaml +++ b/examples/workflows/upstream/synchronization-wf-level.upstream.yaml @@ -16,10 +16,10 @@ metadata: spec: entrypoint: hello-world synchronization: - semaphore: - configMapKeyRef: - name: my-config - key: workflow + semaphores: + - configMapKeyRef: + name: my-config + key: workflow templates: - name: hello-world container: diff --git a/examples/workflows/upstream/synchronization_mutex_tmpl_level.py b/examples/workflows/upstream/synchronization_mutex_tmpl_level_legacy.py similarity index 100% rename from examples/workflows/upstream/synchronization_mutex_tmpl_level.py rename to examples/workflows/upstream/synchronization_mutex_tmpl_level_legacy.py diff --git a/examples/workflows/upstream/synchronization_tmpl_level.py b/examples/workflows/upstream/synchronization_tmpl_level_legacy.py similarity index 100% rename from examples/workflows/upstream/synchronization_tmpl_level.py rename to examples/workflows/upstream/synchronization_tmpl_level_legacy.py diff --git a/examples/workflows/upstream/synchronization_wf_level.py b/examples/workflows/upstream/synchronization_wf_level_legacy.py similarity index 100% rename from examples/workflows/upstream/synchronization_wf_level.py rename to examples/workflows/upstream/synchronization_wf_level_legacy.py diff --git a/examples/workflows/use_cases/dask.yaml b/examples/workflows/use_cases/dask.yaml index 9ad9ed803..e152dc9c8 100644 --- a/examples/workflows/use_cases/dask.yaml +++ b/examples/workflows/use_cases/dask.yaml @@ -35,7 +35,7 @@ spec: import dask.array as da from dask.distributed import Client from dask_kubernetes.operator import KubeCluster - cluster = KubeCluster(image='ghcr.io/dask/dask:latest', resources={"requests": {"memory": "2G", "cpu": "1"}, "limits": {"memory": "4G", "cpu": "1"}}, namespace=namespace, n_workers=n_workers) + cluster = KubeCluster(image='ghcr.io/dask/dask:latest', resources={'requests': {'memory': '2G', 'cpu': '1'}, 'limits': {'memory': '4G', 'cpu': '1'}}, namespace=namespace, n_workers=n_workers) client = Client(cluster) array = da.ones((1000, 1000, 1000)) print('Array mean = {array_mean}, expected = 1.0'.format(array_mean=array.mean().compute())) diff --git a/tests/test_remaining_examples.py b/tests/test_remaining_examples.py index 3a5af67cd..1c4bd0735 100644 --- a/tests/test_remaining_examples.py +++ b/tests/test_remaining_examples.py @@ -24,6 +24,12 @@ "pod-gc-strategy.upstream.yaml", "webhdfs-input-output-artifacts.upstream.yaml", "workflow-template__templates.upstream.yaml", + "synchronization-wf-level.upstream.yaml", + "synchronization-mutex-tmpl-level.upstream.yaml", + "synchronization-mutex-wf-level.upstream.yaml", + "synchronization-tmpl-level.upstream.yaml", + "cron-when.upstream.yaml", + "cron-workflow-multiple-schedules.upstream.yaml", ]