From 98ff420c0bb209c54f3e99fd68167add32c9be85 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Fri, 15 Nov 2024 14:13:47 -0600 Subject: [PATCH 1/2] chore: retire unused container This container and its code aren't used anymore. The code that was used has moved into the workflows python package already. --- .github/workflows/build-container-images.yaml | 12 --- containers/argo_utils/Dockerfile.argo_utils | 25 ------- containers/argo_utils/README.md | 61 ---------------- .../argo_utils/code/argo_python/README.md | 40 ---------- .../argo_utils/code/argo_python/__init__.py | 73 ------------------- docs/component-argo-workflows.md | 15 ---- 6 files changed, 226 deletions(-) delete mode 100644 containers/argo_utils/Dockerfile.argo_utils delete mode 100644 containers/argo_utils/README.md delete mode 100644 containers/argo_utils/code/argo_python/README.md delete mode 100644 containers/argo_utils/code/argo_python/__init__.py diff --git a/.github/workflows/build-container-images.yaml b/.github/workflows/build-container-images.yaml index ea93202d..42ffa11b 100644 --- a/.github/workflows/build-container-images.yaml +++ b/.github/workflows/build-container-images.yaml @@ -5,7 +5,6 @@ on: workflow_dispatch: pull_request: paths: - - "containers/argo_utils/**" - "containers/bmc-utils/**" - "containers/python311_alpine/**" - "containers/python312_alpine/**" @@ -69,17 +68,6 @@ jobs: labels: | org.opencontainers.image.version=${{ env.VERSION_PYTHON312 }} - - name: Build and deploy Argo Utils image - uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6 - with: - context: containers/argo_utils/ - file: containers/argo_utils/Dockerfile.argo_utils - # push for all main branch commits - push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/${{ github.repository }}/argo-utils-python3.11.8:latest,ghcr.io/${{ github.repository }}/argo-utils-python3.11.8:${{ env.VERSION_ARGO_UTILS }} - labels: | - org.opencontainers.image.version=${{ env.VERSION_ARGO_UTILS }} - - name: Build and deploy BMC Utils image uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6 with: diff --git a/containers/argo_utils/Dockerfile.argo_utils b/containers/argo_utils/Dockerfile.argo_utils deleted file mode 100644 index 4241085a..00000000 --- a/containers/argo_utils/Dockerfile.argo_utils +++ /dev/null @@ -1,25 +0,0 @@ -ARG BASE=ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest - -FROM ${BASE} AS builder -RUN --mount=type=cache,target=/root/.cache/.pip pip install --no-cache-dir kubernetes==29.0.0 - -FROM ${BASE} AS prod -ARG APP_PATH=/app -ARG APP_USER=appuser -ARG APP_GROUP=appgroup -ARG APP_USER_UID=1000 -ARG APP_GROUP_GID=1000 -LABEL org.opencontainers.image.title="Python 3.11 image with Argo Utils" -LABEL org.opencontainers.image.base.name="ghcr.io/rackerlabs/understack/argo-utils-python3.11.8" -LABEL org.opencontainers.image.source=https://github.com/rackerlabs/understack - - -ENV PATH="/opt/venv/bin:$PATH" -COPY --from=builder /opt/venv /opt/venv - -WORKDIR /app - -USER $APP_USER - -COPY --chown=${APP_USER}:${APP_GROUP} code/ /app -CMD ["python", "-"] diff --git a/containers/argo_utils/README.md b/containers/argo_utils/README.md deleted file mode 100644 index 61cb41a5..00000000 --- a/containers/argo_utils/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Overview - -The WorkflowTemplates provided in this directory were created to establish commonly used Workflows to be consumed by other, likely larger, Workflows. For example, -the nautobot-api WorkflowTemplate is consumed by the get-device-nautobot WorkflowTemplate, to produce it's own output containing information on a Nautobot Device, -which can then be consumed by future WorkflowTemplates. - -## Setup - -The Nautobot Workflows require a ConfigMap. - -```bash -kubectl -n argo create configmap nautobot '--from-literal=url=https://nautobot.local' -``` - -## Caveats - -Currently Understack is not including any sort of secret store, outside of what is natively provided by Kubernetes. As such, these workflows make a couple of assumptions: - -- Credentials will be mounted from a Kubernetes Secret. -- How those credential Secrets are created is up to you. - -`workflowtemplates/get-bmc-creds.yaml` and the placeholder secret provided in `deps/` are strictly that, placeholder. They have been provided to allow the workflows to execute, -however they will likely fail until a proper get-bmc-creds Workflow is created. - -## Example -```bash -argo -n argo-events submit --from workflowtemplate/get-device-nautobot --parameter device_id=1de4f169-9848-4d8e-921b-65338c1e00ca - -Name: get-device-nautobot-g5wlz -Namespace: argo-events -ServiceAccount: unset -Status: Pending -Created: Tue Apr 23 13:50:57 -0400 (now) -Progress: -Parameters: - device_id: 1de4f169-9848-4d8e-921b-65338c1e00ca -``` - -```bash -argo -n argo-events get @latest - -Name: get-device-nautobot-g5wlz -Namespace: argo-events -ServiceAccount: workflow -Status: Succeeded -Conditions: - PodRunning False - Completed True -Created: Tue Apr 23 13:50:57 -0400 (38 seconds ago) -Started: Tue Apr 23 13:50:57 -0400 (38 seconds ago) -Finished: Tue Apr 23 13:51:27 -0400 (8 seconds ago) -Duration: 30 seconds -Progress: 1/1 -Parameters: - device_id: 1de4f169-9848-4d8e-921b-65338c1e00ca - -STEP TEMPLATE PODNAME DURATION MESSAGE - ✔ get-device-nautobot-g5wlz main - └───✔ nautobot-query nautobot-api/main - └───✔ send-request http -``` diff --git a/containers/argo_utils/code/argo_python/README.md b/containers/argo_utils/code/argo_python/README.md deleted file mode 100644 index a49832d7..00000000 --- a/containers/argo_utils/code/argo_python/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# argo_python - -A quick helper method to write kubernetes secrets. This accomplishes a couple things. - -1) Allows us to share sensitive data to subsequent steps in the workflow -2) Sets metadata.ownerReferences to the workflow where it is cleaned up once the workflow is - -note: to add the ownerReference for garbage collection, the Pod's uid needs to be obtained. This can either be done by -providing get permission on the pods resource for the service account running this Pod, or by passing the Pod uid into -the container with something like: - -```yaml - env: - - name: WF_UID - value: "{{workflow.uid}}" - - name: WF_NAME - value: "{{workflow.name}}" - - name: WF_NS - value: "{{workflow.namespace}}" -``` - -### Example -```python - - from argo_python import ArgoWorkflow - import base64 - import os - import uuid - - data = { - 'username': base64.b64encode("".encode("utf-8")).decode(), - 'password': base64.b64encode("".encode("utf-8")).decode(), - } - - workflow = ArgoWorkflow(os.getenv("WF_NS"), os.getenv("WF_NAME"), os.getenv("WF_UID")) - secret_name = workflow.create_secret(f"creds-{uuid.uuid4()}", data) - print(secret_name) - - # example output: "creds-97794d2b-338c-44c6-8587-e3086e6a1bf7" -``` diff --git a/containers/argo_utils/code/argo_python/__init__.py b/containers/argo_utils/code/argo_python/__init__.py deleted file mode 100644 index 091af98e..00000000 --- a/containers/argo_utils/code/argo_python/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -from kubernetes import client, config -from typing import Dict -import logging - -logger = logging.getLogger(__name__) - - -class ArgoWorkflow: - def __init__( - self, - namespace: str, - name: str, - uid: str, - api_version="argoproj.io/v1alpha1", - config_file=None, - ): - if config_file: - config.load_kube_config(config_file) - else: - config.load_incluster_config() - - self.kube = client.CoreV1Api() - self.namespace = namespace - self.name = name - self.uid = uid - self.api_version = api_version - - def get_pod(self, name): - r = self.kube.read_namespaced_pod(name, self.namespace) - return r.metadata - - def create_secret(self, name: str, data: Dict) -> str: - """ - create_secret creates a kubernetes secret, setting ownerReferences to provide garbage collection. - - :param name: metadata.name of the kubernetes secret - :param data: data dict to be stored in the secret - :return: the name of the kubernetes secret - """ - - owner = client.V1OwnerReference( - api_version=self.api_version, - kind="Workflow", - name=self.name, - uid=self.uid, - block_owner_deletion=True, - controller=True, - ) - - meta = client.V1ObjectMeta( - name=name, - owner_references=[owner], - ) - - secret = client.V1Secret( - api_version="v1", - kind="Secret", - type="Opaque", - metadata=meta, - data=data, - ) - - # create or update secret - try: - self.kube.read_namespaced_secret(name, self.namespace) - r = self.kube.patch_namespaced_secret(name, self.namespace, secret) - return r.metadata.name - except client.exceptions.ApiException as e: - if e.status == 404: - r = self.kube.create_namespaced_secret(self.namespace, secret) - return r.metadata.name - else: - raise diff --git a/docs/component-argo-workflows.md b/docs/component-argo-workflows.md index b98d25f7..8f330b97 100644 --- a/docs/component-argo-workflows.md +++ b/docs/component-argo-workflows.md @@ -39,20 +39,6 @@ Authorization is handled by Kubernetes' RBAC services. Workflows are run with th Account. We've provided the `workflow` Service Account, which has been granted access to the necessary Kubernetes resources, to run the provided WorkflowTemplates. -### argo-python - -To facilitate the ability to pass data securely between Workflows the [argo-python][argo-python] Class was written. -This Python Class writes Kubernetes Secrets directly to the Kubernetes API from the Workflow's Pod, allowing these -Secrets to be securely mounted into a subsequent Workflow's environment. - -By default these Secrets are created with an ownerReference set to the Workflow which created them, which allows -them to be garbage collected when that Workflow is terminated. The ownerReference requires details about the -Workflow to be passed in. This is available in the Workflow from the -[workflow variable](https://argo-workflows.readthedocs.io/en/latest/variables/#global). - -An example WorkflowTemplate demonstrating argo-python usage can be found -[here](https://github.com/rackerlabs/understack/blob/main/workflows/argo-events/workflowtemplates/get-bmc-creds.yaml). - ### Argo CLI Argo Workflows has a CLI and the installation instructions can be found [here](https://github.com/argoproj/argo-workflows/releases/). @@ -98,4 +84,3 @@ STEP TEMPLATE PODNAME DURATION MESSAGE [argo-workflows]: [argo-events]: -[argo-python]: From 0008ad244bc38c10341cece66a5e2cbc2893fe5c Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Fri, 15 Nov 2024 18:04:14 -0600 Subject: [PATCH 2/2] chore: remove unused workflow --- workflows/argo-events/kustomization.yaml | 1 - .../workflowtemplates/node-events.yaml | 54 ------------------- 2 files changed, 55 deletions(-) delete mode 100644 workflows/argo-events/workflowtemplates/node-events.yaml diff --git a/workflows/argo-events/kustomization.yaml b/workflows/argo-events/kustomization.yaml index 38ce4c45..d93d2469 100644 --- a/workflows/argo-events/kustomization.yaml +++ b/workflows/argo-events/kustomization.yaml @@ -17,7 +17,6 @@ resources: - serviceaccounts/openstack-sensor-submit-workflow.yaml - workflowtemplates/nautobot-api.yaml - workflowtemplates/sync-provision-state-to-nautobot.yaml - - workflowtemplates/node-events.yaml - workflowtemplates/undersync-device.yaml - workflowtemplates/undersync-switch.yaml - workflowtemplates/keystone-event-project.yaml diff --git a/workflows/argo-events/workflowtemplates/node-events.yaml b/workflows/argo-events/workflowtemplates/node-events.yaml deleted file mode 100644 index a2a12ca5..00000000 --- a/workflows/argo-events/workflowtemplates/node-events.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: node-events - annotations: - workflows.argoproj.io/description: | - Defined in `workflows/argo-events/workflowtemplates/node-events.yaml` -spec: - volumes: - - name: argo-secrets - secret: - secretName: operate-workflow-sa.token - entrypoint: publish-node-updated - templates: - - name: publish-node-updated - inputs: - parameters: - - name: url - value: https://argo-server.argo.svc.cluster.local:2746/api/v1/events/argo/ - - name: body - value: '{}' - script: - image: dot-registry.ohthree.com/doug1840/argo-python:latest - command: [python] - volumeMounts: - - mountPath: /etc/argo/ - name: argo-secrets - readOnly: true - source: | - import requests - import sys - import logging - import urllib3 - - urllib3.disable_warnings() - - logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger(__name__) - - URL = "{{inputs.parameters.url}}" - - with open("/etc/argo/token", "r") as secret: - auth_secret = secret.read() - - request_body = '{{inputs.parameters.body}}' - logger.info(f"Sending event to {URL}") - response = requests.post(URL, - request_body, - headers={"Authorization": f"Bearer {auth_secret}"}, - verify=False - ) - logger.info(response) - if response.status_code != 200: - sys.exit(1)