Merge branch 'main' into extract-report-logic

.github/workflows/container-build-push.yaml

@@ -1,3 +1,4 @@
+---
 name: Container Build and Push
 
 on:
@@ -8,15 +9,145 @@ on:
       - v*
   pull_request:
 
+defaults:
+  run:
+    shell: bash
+
 permissions:
   contents: read
   packages: write
-  # This is used to complete the identity challenge
-  # with sigstore/fulcio.
+  # This is used to complete the identity challenge with sigstore/fulcio.
   id-token: write
 
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   build-push:
-    uses: darbiadev/.github/.github/workflows/docker-build-push.yaml@e3ebedcaeee8d40bdb7ef569dacd74829ab0c368 # v14.0.0
-    with:
-      file-name: Dockerfile
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938  # v4.2.0
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382  # v3.6.0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db  # v3.6.1
+        with:
+          platforms: ${{ matrix.platform }}
+
+      - name: Log in to container registry (${{ env.REGISTRY }})
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567  # v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: docker_meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81  # v5.5.1
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=edge
+            # FIXME: Remove explicit `latest` tag once we start tagging releases
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=tag
+            type=sha,format=long
+
+      - name: Build and push Docker image
+        id: docker_build_push
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75  # v6.9.0
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            git_sha=${{ github.sha }}
+          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.ref == 'refs/heads/main' || startswith(github.event.ref, 'refs/tags/v') }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+
+      # Sign the resulting Docker image digest.
+      # This will only write to the public Rekor transparency log when the Docker repository is public to avoid leaking
+      # data. If you would like to publish transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.ref == 'refs/heads/main' || startswith(github.event.ref, 'refs/tags/v') }}
+        # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio
+        # instance.
+        run: cosign sign --yes ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.docker_build_push.outputs.digest }}
+
+      - name: Export digest
+        if: ${{ github.ref == 'refs/heads/main' || startswith(github.event.ref, 'refs/tags/v') }}
+        run: |
+          mkdir -p /tmp/digests
+          digest='${{ steps.docker_build_push.outputs.digest }}'
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: ${{ github.ref == 'refs/heads/main' || startswith(github.event.ref, 'refs/tags/v') }}
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874  # v4.4.0
+        with:
+          if-no-files-found: error
+          name: digests
+          path: /tmp/digests/*
+          retention-days: 1
+
+  merge:
+    if: ${{ github.ref == 'refs/heads/main' || startswith(github.event.ref, 'refs/tags/v') }}
+    needs:
+      - build-push
+
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16  # v4.1.8
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db  # v3.6.1
+
+      - name: Log in to container registry (${{ env.REGISTRY }})
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567  # v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: docker_meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81  # v5.5.1
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=edge
+            # FIXME: Remove explicit `latest` tag once we start tagging releases
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=tag
+            type=sha,format=long
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: >
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "${DOCKER_METADATA_OUTPUT_JSON}") \
+            $(printf ' ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: >-
+          docker buildx imagetools inspect \
+            '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.docker_meta.outputs.version }}'

.github/workflows/generate-migration-sql.yaml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout (base)
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938  # v4.2.0
         with:
           ref: ${{ github.event.pull_request.base.sha }}
 
@@ -50,7 +50,7 @@ jobs:
           echo "BASE_MIGRATION_REVISION=${base_head}" >>"${GITHUB_ENV}"
 
       - name: Checkout (HEAD)
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938  # v4.2.0
         with:
           clean: false
           ref: ${{ github.event.pull_request.head.sha }}

.github/workflows/lint-test.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
       - name: "Setup PDM"
         uses: pdm-project/setup-pdm@568ddd69406b30de1774ec0044b73ae06e716aa4 # v4

.github/workflows/publish-docs.yaml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
       - name: "Setup PDM"
         uses: pdm-project/setup-pdm@568ddd69406b30de1774ec0044b73ae06e716aa4 # v4

.github/workflows/sentry-release.yaml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
       - name: "Create Sentry release"
         uses: getsentry/action-release@e769183448303de84c5a06aaaddf9da7be26d6c7 # v1.7.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c as builder
+FROM python:3.12-slim@sha256:af4e85f1cac90dd3771e47292ea7c8a9830abfabbe4faa5c53f158854c2e819d as builder
 
 RUN pip install -U pip setuptools wheel
 RUN pip install pdm
@@ -22,7 +22,7 @@ ENV GIT_SHA="testing"
 
 CMD ["pdm", "run", "coverage"]
 
-FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c as prod
+FROM python:3.12-slim@sha256:af4e85f1cac90dd3771e47292ea7c8a9830abfabbe4faa5c53f158854c2e819d as prod
 
 # Define Git SHA build argument for sentry
 ARG git_sha="development"

pyproject.toml

@@ -23,6 +23,8 @@ dependencies = [
   "structlog-sentry==2.1.0",
   "structlog==24.4.0",
   "uvicorn[standard]==0.30.3",
+  "prometheus-client>=0.21.0",
+  "prometheus-fastapi-instrumentator>=7.0.0",
 ]
 dynamic = ["version"]
 

src/mainframe/endpoints/package.py

@@ -24,6 +24,8 @@
     QueuePackageResponse,
 )
 
+from mainframe.metrics import packages_ingested, packages_in_queue, packages_fail, packages_success
+
 router = APIRouter(tags=["package"])
 logger: structlog.stdlib.BoundLogger = structlog.get_logger()
 
@@ -62,6 +64,8 @@ def submit_results(
         log.error(
             f"Scan {name}@{version} already in a FINISHED state", error_message=error.detail, tag="already_finished"
         )
+        packages_fail.inc()
+        packages_in_queue.dec()
         raise error
 
     with session, session.begin():
@@ -104,6 +108,9 @@ def submit_results(
         tag="scan_submitted",
     )
 
+    packages_success.inc()
+    packages_in_queue.dec()
+
 
 @router.get(
     "/package",
@@ -226,6 +233,9 @@ def batch_queue_package(
 
             session.add(scan)
 
+            packages_ingested.inc()
+            packages_in_queue.inc()
+
 
 @router.post(
     "/package",
@@ -294,4 +304,7 @@ def queue_package(
         tag="package_added",
     )
 
+    packages_ingested.inc()
+    packages_in_queue.inc()
+
     return QueuePackageResponse(id=str(new_package.scan_id))

src/mainframe/endpoints/report.py

@@ -20,6 +20,8 @@
     ReportPackageBody,
 )
 
+from mainframe.metrics import packages_reported
+
 logger: structlog.stdlib.BoundLogger = structlog.get_logger()
 
 
@@ -256,3 +258,5 @@ def report_package(
         },
         reported_by=auth.subject,
     )
+
+    packages_reported.inc()

src/mainframe/metrics.py

@@ -0,0 +1,14 @@
+from prometheus_client import Counter, Gauge
+
+
+packages_ingested = Counter("packages_ingested", "Total number of packages ingested")
+
+packages_in_queue = Gauge(
+    "packages_in_queue",
+    "Packages that are currently waiting to be scanned. Includes queued and pending packages.",
+)
+
+packages_success = Counter("packages_success", "Number of packages scanned successfully")
+packages_fail = Counter("packages_fail", "Number of packages that failed scanning")
+
+packages_reported = Counter("packages_reported", "Number of packages reported")

src/mainframe/server.py

@@ -20,6 +20,8 @@
 from mainframe.models.schemas import ServerMetadata
 from mainframe.rules import Rules, fetch_rules
 
+from prometheus_fastapi_instrumentator import Instrumentator
+
 from . import __version__
 
 
@@ -72,6 +74,8 @@ async def lifespan(app_: FastAPI):
     version=__version__,
 )
 
+Instrumentator().instrument(app).expose(app)
+
 if GIT_SHA in ("development", "testing"):
     app.dependency_overrides[validate_token] = validate_token_override