Migrate CI from AWS queues to Google Kubernetes Engine queues

- Add necessary k8s plugin config to the pipeline for use with GKE + Buildkite's agent-stack-k8s helm stack - Remove docker agent tag filter - Install bk agent in Dockerfile for use in the code coverage step
uber-go · Apr 10, 2024 · 9145f8b · 9145f8b
1 parent 3ca6328
commit 9145f8b
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 23 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,81 +1,147 @@
+container:
+  kubernetes: &kubernetes
+    sidecars:
+    - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
+      command:
+      - |-
+        dockerd-entrypoint.sh
+      volumeMounts:
+        - mountPath: /var/run/
+          name: docker-sock
+      securityContext:
+        privileged: true
+        allowPrivilegeEscalation: true
+    mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars
+    podSpec: &podSpec
+      containers:
+        - &commandContainer
+          image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2
+          command:
+          - |-
+            echo "Command step was not overridden."
+            exit 1
+          volumeMounts:
+            - mountPath: /var/run/
+              name: docker-sock
+          resources:
+            requests:
+              cpu: 7500m
+              memory: 28G
+      volumes:
+      - name: docker-sock
+        emptyDir: {}
+
 steps:
   - label: "fossa analyze"
     agents:
-      queue: "init"
-      docker: "*"
+      queue: "buildkite-gcp-medium-helm"
     command: ".buildkite/scripts/fossa.sh"
 
   - label: ":golang: unit-test"
     agents:
-      queue: "workers"
-      docker: "*"
-    commands:
-      - "make unit_test"
-      - ".buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt"
+      queue: "buildkite-gcp-large-helm"
     artifact_paths:
       - ".build/*/coverage/*.out"
       - ".build/cover.out"
       - ".build/metadata.txt"
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make unit_test
+                  .buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt
       - docker-compose#v3.0.0:
           run: unit-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golangci-lint: validate code is clean"
     agents:
-      queue: "workers"
-      docker: "*"
-    command: "./scripts/golint.sh"
+      queue: "buildkite-gcp-large-helm"
     artifact_paths: [ ]
     retry:
       automatic:
         limit: 1
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  ./scripts/golint.sh
       - docker-compose#v3.0.0:
           run: unit-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-sticky-off"
     agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_sticky_off"
+      queue: "buildkite-gcp-large-helm"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
         limit: 1
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_sticky_off
       - docker-compose#v3.0.0:
           run: integ-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-sticky-on"
     agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_sticky_on"
+      queue: "buildkite-gcp-large-helm"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
         limit: 1
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_sticky_on
       - docker-compose#v3.0.0:
           run: integ-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-grpc-adapter"
     agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_grpc"
+      queue: "buildkite-gcp-large-helm"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
         limit: 1
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_grpc
       - docker-compose#v3.0.0:
           run: integ-test-grpc
           config: docker/buildkite/docker-compose.yml
@@ -84,10 +150,17 @@ steps:
 
   - label: ":golang: code-coverage"
     agents:
-      queue: "workers"
-      docker: "*"
-    command: ".buildkite/scripts/gocov.sh"
+      queue: "buildkite-gcp-large-helm"
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  .buildkite/scripts/gocov.sh
       - docker-compose#v3.0.0:
           run: coverage-report
           config: docker/buildkite/docker-compose.yml
diff --git a/docker/buildkite/Dockerfile b/docker/buildkite/Dockerfile
@@ -8,3 +8,18 @@ ADD go.mod go.sum /go/src/go.uber.org/cadence/
 # allow git-status and similar to work
 RUN git config --global --add safe.directory /go/src/go.uber.org/cadence
 RUN GO111MODULE=on go mod download
+
+# Install dependencies needed to install buildkite-agent
+RUN apt-get install -y apt-transport-https dirmngr curl
+
+# Add Buildkite's GPG key
+RUN curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | \
+  gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg
+
+# Add the repository to Apt sources
+RUN echo \
+  "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | \
+  tee /etc/apt/sources.list.d/buildkite-agent.list
+
+# Install Buildkite agent
+RUN apt-get update && apt-get install -yy --no-install-recommends buildkite-agent
diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml
@@ -116,7 +116,6 @@ services:
       - "GO111MODULE=on"
     volumes:
       - ../../:/go/src/go.uber.org/cadence
-      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
 
 networks:
   services-network: