feat: Reorganise workflow yamls

.github/workflows/build-container-images.yaml

@@ -5,16 +5,18 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'argo-workflows/*/containers/*'
-      - 'argo-workflows/*/code/**'
-      - '!argo-workflows/example-feature/**'
+      - "containers/argo_utils/**"
+      - "containers/obm-utils/**"
+      - "containers/python311_alpine/**"
+      - "containers/python312_alpine/**"
   push:
     branches:
       - main
     paths:
-      - 'argo-workflows/*/containers/*'
-      - 'argo-workflows/*/code/**'
-      - '!argo-workflows/example-feature/**'
+      - "containers/argo_utils/**"
+      - "containers/obm-utils/**"
+      - "containers/python311_alpine/**"
+      - "containers/python312_alpine/**"
 
 # bump container versions here, they will be populated to tags and labels
 env:
@@ -46,43 +48,43 @@ jobs:
       - name: Build and deploy Python 3.11 image
         uses: docker/build-push-action@v5
         with:
-          context: argo-workflows/generic/
-          file: argo-workflows/generic/containers/Dockerfile.python311_alpine
+          context: containers/python311_alpine/
+          file: containers/python311_alpine/Dockerfile.python311_alpine
           # push for all main branch commits
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest,ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:${{ env.VERSION_PYTHON311 }}
+          tags: ghcr.io/${{ github.repository }}/argo-python3.11.8-alpine3.19:latest,ghcr.io/${{ github.repository }}/argo-python3.11.8-alpine3.19:${{ env.VERSION_PYTHON311 }}
           labels: |
               org.opencontainers.image.version=${{ env.VERSION_PYTHON311 }}
 
       - name: Build and deploy Python 3.12 image
         uses: docker/build-push-action@v5
         with:
-          context: argo-workflows/generic/
-          file: argo-workflows/generic/containers/Dockerfile.python312_alpine
+          context: containers/python312_alpine/
+          file: containers/python312_alpine/Dockerfile.python312_alpine
           # push for all main branch commits
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ghcr.io/rackerlabs/understack/argo-python3.12.2-alpine3.19:latest,ghcr.io/rackerlabs/understack/argo-python3.12.2-alpine3.19:${{ env.VERSION_PYTHON312 }}
+          tags: ghcr.io/${{ github.repository }}/argo-python3.12.2-alpine3.19:latest,ghcr.io/${{ github.repository }}/argo-python3.12.2-alpine3.19:${{ env.VERSION_PYTHON312 }}
           labels: |
               org.opencontainers.image.version=${{ env.VERSION_PYTHON312 }}
 
       - name: Build and deploy Argo Utils image
         uses: docker/build-push-action@v5
         with:
-          context: argo-workflows/generic/
-          file: argo-workflows/generic/containers/Dockerfile.argo_utils
+          context: containers/argo_utils/
+          file: containers/argo_utils/Dockerfile.argo_utils
           # push for all main branch commits
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ghcr.io/rackerlabs/understack/argo-utils-python3.11.8:latest,ghcr.io/rackerlabs/understack/argo-utils-python3.11.8:${{ env.VERSION_ARGO_UTILS }}
+          tags: ghcr.io/${{ github.repository }}/argo-utils-python3.11.8:latest,ghcr.io/${{ github.repository }}/argo-utils-python3.11.8:${{ env.VERSION_ARGO_UTILS }}
           labels: |
               org.opencontainers.image.version=${{ env.VERSION_ARGO_UTILS }}
 
       - name: Build and deploy OBM Utils image
         uses: docker/build-push-action@v5
         with:
-          context: argo-workflows/obm-utils/
-          file: argo-workflows/obm-utils/containers/Dockerfile.obm_utils
+          context: containers/obm-utils/
+          file: containers/obm-utils/Dockerfile.obm_utils
           # push for all main branch commits
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ghcr.io/rackerlabs/understack/argo-obm-utils-python3.11.8:latest,ghcr.io/rackerlabs/understack/argo-obm-utils-python3.11.8:${{ env.VERSION_OBM_UTILS }}
+          tags: ghcr.io/${{ github.repository }}/argo-obm-utils-python3.11.8:latest,ghcr.io/${{ github.repository }}/argo-obm-utils-python3.11.8:${{ env.VERSION_OBM_UTILS }}
           labels: |
               org.opencontainers.image.version=${{ env.VERSION_OBM_UTILS }}

.github/workflows/containers.yaml

@@ -53,10 +53,10 @@ jobs:
           # that are arch specific so populate them at the index as well.
           DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
 
-      - name: build and deploy container image to registry
+      - name: build and deploy openstack container image to registry
         uses: docker/build-push-action@v5
         with:
-          file: containers/Dockerfile.${{ matrix.project }}
+          file: containers/${{ matrix.project }}/Dockerfile.${{ matrix.project }}
           build-args: OPENSTACK_VERSION=${{ matrix.openstack }}
           pull: true  # ensure we always have an up to date source
           push: true
@@ -98,8 +98,9 @@ jobs:
       - name: build and deploy dnsmasq container to registry
         uses: docker/build-push-action@v5
         with:
+          context: "{{defaultContext}}:containers/dnsmasq"
           file: containers/Dockerfile.dnsmasq
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
@@ -142,7 +143,7 @@ jobs:
       - name: build and deploy container image to registry
         uses: docker/build-push-action@v5
         with:
-          file: containers/Dockerfile.${{ matrix.container.name }}
+          file: containers/${{ matrix.container.name }}/Dockerfile.${{ matrix.container.name }}
           pull: true  # ensure we always have an up to date source
           push: true
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/yamllint.yaml

@@ -41,12 +41,15 @@ jobs:
       - uses: actions/checkout@v4
       - uses: thiagodnf/[email protected]
         with:
-          jsonSchemaFile: argo-workflows/.workflow_with_description.schema.json
+          jsonSchemaFile: workflows/.workflow_with_description.schema.json
           yamlFiles: |
-            argo-workflows/**/workflowtemplates/*.y*ml
-            argo-workflows/**/sensors/*.y*ml
-            argo-workflows/**/workflows/*.y*ml
-            apps/understack-workflows/workflows/workflowtemplates/*.y*ml
+            workflows/argo-events/secrets/*.y*ml
+            workflows/argo-events/sensors/*.y*ml
+            workflows/argo-events/eventsources/*.y*ml
+            workflows/argo-events/roles/*.y*ml
+            workflows/argo-events/rolebindings/*.y*ml
+            workflows/argo-events/workflowtemplates/*.y*ml
+            workflows/openstack/*.y*ml
 
   shellcheck:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
         args:
           - --schemafile
           - https://raw.githubusercontent.com/argoproj/argo-workflows/master/api/jsonschema/schema.json
-        files: "argo-workflows/.*/(workflows|workflowtemplates)/.*.(yml|yaml)$"
+        files: "workflows/argo-events/(sensors|workflowtemplates)/.*.(yml|yaml)$"
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.37.0
     hooks:

Makefile

@@ -28,7 +28,7 @@ $(ACTIVATE): requirements-docs.txt
 
 docs/workflows: $(ACTIVATE)
 	@mkdir -p docs/workflows
-	@$(PYTHON) scripts/argo-workflows-to-mkdocs.py argo-workflows docs/workflows
+	@$(PYTHON) scripts/argo-workflows-to-mkdocs.py workflows docs/workflows
 
 .PHONY: docs
 docs: $(ACTIVATE) docs/workflows ## Builds the documentation

apps/appsets/components.yaml

@@ -179,7 +179,7 @@ spec:
                   source:
                     repoURL: '{{index .metadata.annotations "uc_repo_git_url"}}'
                     targetRevision: '{{index .metadata.annotations "uc_repo_ref"}}'
-                    path: 'apps/understack-workflows'
+                    path: 'workflows'
             selector:
               # by setting the key in the elements 'skipComponent' to 'true' it will skip installing it
               # ArgoCD's templating operates with strings so it's the string "true"

apps/appsets/workflows.yaml

@@ -20,11 +20,10 @@ spec:
       sources:
         - repoURL: '{{index .metadata.annotations "uc_repo_git_url"}}'
           targetRevision: '{{index .metadata.annotations "uc_repo_ref"}}'
-          path: 'argo-workflows'
+          path: 'workflows/argo-events'
           directory:
             recurse: true
-            include: '{generic/deps/*.yaml,shared-sensors/*.yaml,*/sensors/*.yaml,*/workflowtemplates/*.yaml}'
-            exclude: '{example-feature/*,*/*/*example*}'
+            include: '{roles/*.yaml,rolebindings/*.yaml,secrets/*.yaml,sensors/*yaml,eventsources/*.yaml,workflowtemplates/*.yaml}'
         - repoURL: '{{index .metadata.annotations "uc_deploy_git_url"}}'
           targetRevision: '{{index .metadata.annotations "uc_deploy_ref"}}'
           path: 'argo-workflows'

containers/Dockerfile.dnsmasq → containers/dnsmasq/Dockerfile.dnsmasq

@@ -42,7 +42,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/*.deb
 
-COPY containers/common/helpers.sh /helpers.sh
+COPY containers/dnsmasq/common/helpers.sh /helpers.sh
 COPY containers/dnsmasq/entry-point.sh /entry-point.sh
 RUN chmod +x /entry-point.sh
 COPY containers/dnsmasq/dnsmasq.conf.j2 /etc/dnsmasq.conf.j2

argo-workflows/obm-utils/containers/Dockerfile.obm_utils → containers/obm-utils/Dockerfile.obm_utils

@@ -8,7 +8,7 @@ ARG APP_GROUP=appgroup
 ARG APP_USER_UID=1000
 ARG APP_GROUP_GID=1000
 
-COPY --chown=${APP_USER}:${APP_GROUP} containers/requirements.txt /app
+COPY --chown=${APP_USER}:${APP_GROUP} requirements.txt /app
 RUN --mount=type=cache,target=/root/.cache/.pip pip install --no-cache-dir -r /app/requirements.txt
 
 FROM ${BASE} as prod

docs/component-argo-workflows.md

@@ -53,7 +53,7 @@ the Pod's uid can be passed via the `KUBERNETES_POD_UID` environment variable. T
 completion of the Workflow `.spec.podGC.strategy` can be set to `OnWorkflowCompletion`.
 
 An example WorkflowTemplate demonstrating argo-python usage can be found
-[here](https://github.com/rackerlabs/understack/blob/main/argo-workflows/generic/workflowtemplates/get-obm-creds.yaml).
+[here](https://github.com/rackerlabs/understack/blob/main/workflows/argo-events/workflowtemplates/get-obm-creds.yaml).
 
 ### Argo CLI
 

python/understack-workflows/understack_workflows/main/undersync_device.py

@@ -83,11 +83,11 @@ def argument_parser():
 
 
 def main():
-    """Updates Interface status in Nautobot triggers Undersync.
+    """Updates Interface Status in Nautobot and triggers Undersync.
 
-    Updates Nautobot Interfaces's status field and follows with request to
-    Undersync service, requesting sync for all of the uplink_switches that the
-    device is connected to.
+    Updates Nautobot Device Interface status field and follows with
+    request to Undersync service, requesting sync for all of the
+    uplink_switches that the device is connected to.
     """
     args = argument_parser().parse_args()
 

workflows/argo-events/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# this is where our workflows currently run
+namespace: argo-events
+
+resources:
+  - secrets/openstack-svc-acct.yaml
+  - secrets/obm-creds.yaml
+  - secrets/placeholder-obm-creds.yaml
+  - secrets/operate-workflow-sa.token.yaml
+  - secrets/production-ironic-for-argo-creds.yaml
+  - secrets/nautobot-token.yaml
+  - secrets/placeholder-obm-legacy-passwords.yaml
+  - sensors/ironic-node-update.yaml
+  - sensors/debug-sensor.yaml
+  - sensors/nb-oob-interface-update.yaml
+  - eventsources/nautobot-webhook.yaml
+  - roles/sensor-submit-workflow-role.yaml
+  - rolebindings/openstack-sensor-submit-workflow.yaml
+  - workflowtemplates/nautobot-api.yaml
+  - workflowtemplates/get-device-nautobot.yaml
+  - workflowtemplates/sync-interfaces-to-nautobot.yaml
+  - workflowtemplates/sync-nb-server-to-ironic.yaml
+  - workflowtemplates/get-obm-ip.yaml
+  - workflowtemplates/sync-provision-state-to-nautobot.yaml
+  - workflowtemplates/node-events.yaml
+  - workflowtemplates/sync-interfaces-to-ironic.yaml
+  - workflowtemplates/undersync-device.yaml
+  - workflowtemplates/sync-server-to-ironic.yaml
+  - workflowtemplates/undersync-switch.yaml
+  - workflowtemplates/keystone-event-project.yaml
+  - workflowtemplates/get-obm-creds.yaml
+  - workflowtemplates/idrac-enable-network-boot.yaml
+  - workflowtemplates/obm-sync-creds.yaml
+  - workflowtemplates/sync-srv-redfish-intfs-to-nb.yaml
+  - workflowtemplates/obm-firmware-update.yaml
+  - workflowtemplates/sync-obm-creds.yaml

workflows/argo-events/rolebindings/openstack-sensor-submit-workflow.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: openstack-sensor-submit-workflow
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: sensor-submit-workflow-role
+subjects:
+  - kind: ServiceAccount
+    name: sensor-submit-workflow
+    namespace: openstack

apps/understack-workflows/workflows/sensor-submit-rbac.yaml → workflows/argo-events/roles/sensor-submit-workflow-role.yaml

@@ -25,16 +25,3 @@ rules:
       - patch
     resources:
       - workflows
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: openstack-sensor-submit-workflow
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: sensor-submit-workflow-role
-subjects:
-  - kind: ServiceAccount
-    name: sensor-submit-workflow
-    namespace: openstack

argo-workflows/sync-nb-server-to-ironic/sensors/debug-sensor.yaml → workflows/argo-events/sensors/debug-sensor.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: argo-events
   annotations:
     workflows.argoproj.io/description: |
-      Defined in `argo-workflows/sync-nb-server-to-ironic/sensors/debug-sensor.yaml`
+      Defined in `workflows/argo-events/sensors/debug-sensor.yaml`
 spec:
   dependencies:
   - eventName: interface-update

argo-workflows/ironic-to-nautobot-sync/sensors/ironic-to-nautobot-sensor.yaml → workflows/argo-events/sensors/ironic-node-update.yaml

@@ -10,7 +10,7 @@ metadata:
   namespace: argo-events
   annotations:
     workflows.argoproj.io/description: |
-      Defined in `argo-workflows/ironic-to-nautobot-sensor/sensors/sync.yaml`
+      Defined in `workflows/argo-events/sensors/ironic-node-update.yaml`
 spec:
   dependencies:
   - eventName: openstack