From c988738a1bf6bec5604bdeaa6f2d929f379ff7f5 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Thu, 28 Jul 2022 13:29:01 -0700 Subject: [PATCH 1/3] Run Agones tests in CI This makes it such that on each PR, the Agones integration tests are run against the GKE + Agones cluster specified in ./agones/main.tf This includes: * Fix for main.tf to align release branch with Agones version. * Configure how long to wait before cleaning up past test namespaces. * Cleanup and optimisation of cloudbuild.yaml. Work on #510 --- agones/main.tf | 2 +- agones/src/lib.rs | 22 +++++++++++++++++++--- build/Makefile | 21 ++++++++++++++++----- cloudbuild.yaml | 45 +++++++++++++++++++++++++++++++++++---------- 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/agones/main.tf b/agones/main.tf index 2ad812d506..dad4de0753 100644 --- a/agones/main.tf +++ b/agones/main.tf @@ -68,7 +68,7 @@ module "agones_cluster" { module "helm_agones" { source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/helm3/?ref=release-1.24.0" - agones_version = "1.23.0" + agones_version = "1.24.0" values_file = "" chart = "agones" host = module.agones_cluster.host diff --git a/agones/src/lib.rs b/agones/src/lib.rs index 61064dfb84..73081e3d49 100644 --- a/agones/src/lib.rs +++ b/agones/src/lib.rs @@ -20,7 +20,9 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use k8s_openapi::{api::core::v1::Namespace, apimachinery::pkg::apis::meta::v1::ObjectMeta}; +use k8s_openapi::{ + api::core::v1::Namespace, apimachinery::pkg::apis::meta::v1::ObjectMeta, chrono, +}; use kube::{ api::{DeleteParams, ListParams, PostParams}, Api, ResourceExt, @@ -33,6 +35,7 @@ mod pod; static CLIENT: OnceCell = OnceCell::const_new(); #[allow(dead_code)] const IMAGE_TAG: &str = "IMAGE_TAG"; +const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS"; pub struct Client { /// The Kubernetes client @@ -75,10 +78,23 @@ async fn setup_namespace(client: kube::Client) -> String { let lp = ListParams::default().labels("owner=quilkin-test"); let nss = namespaces.list(&lp).await.unwrap(); let dp = DeleteParams::default(); + + let delay = env::var_os(DELETE_DELAY_SECONDS) + .map(|value| chrono::Duration::seconds(value.into_string().unwrap().parse().unwrap())); + for ns in nss { let name = ns.name(); - if let Err(err) = namespaces.delete(name.as_str(), &dp).await { - println!("Failure attempting to deleted namespace: {:?}, {err}", name); + + let delete = delay + .and_then(|duration| { + let expiry = ns.creation_timestamp()?.0 + duration; + Some(chrono::Utc::now() > expiry) + }) + .unwrap_or(true); + if delete { + if let Err(err) = namespaces.delete(name.as_str(), &dp).await { + println!("Failure attempting to deleted namespace: {:?}, {err}", name); + } } } diff --git a/build/Makefile b/build/Makefile index d49bd1e292..a5d7188992 100644 --- a/build/Makefile +++ b/build/Makefile @@ -49,8 +49,8 @@ helm_config := ~/.config/helm helm_cache := ~/.cache/helm kube_mount_args := -v $(kubeconfig_path):/root/.kube -v $(helm_config):/root/.config/helm -v $(helm_cache):/root/.cache/helm - minikube_args := --network=host -v ~/.minikube:$(HOME)/.minikube +gcloud_mount_args := -v $(build_path)/.config/gcloud:/root/.config/gcloud # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ @@ -165,8 +165,12 @@ run-test-agones: # Convenience target to build and push quilkin images to a repository. # Use `REPOSITORY` arg to specify the repository to push to. +# USe `SKIP_BUILD_IMAGE` if you want to skip building the image if it has been already built. # See `build-image` for more details. +push: +ifndef SKIP_BUILD_IMAGE push: build-image +endif docker push $(IMAGE_TAG)-debug docker push $(IMAGE_TAG) @@ -196,10 +200,9 @@ docs: ensure-build-image # Start an interactive shell inside the build image # Useful for testing, or adhoc cargo, gcloud, kubectl or terraform commands -shell: ensure-kube-dirs ensure-build-image - -mkdir -p $(build_path)/.gcloud +shell: ensure-gcloud-dirs ensure-kube-dirs ensure-build-image docker run --rm -it $(DOCKER_RUN_ARGS) $(common_rust_args) \ - -v $(build_path)/.config/gcloud:/root/.config/gcloud $(kube_mount_args) \ + $(gcloud_mount_args) $(kube_mount_args) \ --entrypoint=bash $(BUILD_IMAGE_TAG) ensure-build-image: ensure-cargo-registry @@ -208,6 +211,14 @@ ensure-build-image: ensure-cargo-registry ensure-cargo-registry: -mkdir -p $(CARGO_HOME)/registry +ensure-gcloud-dirs: + -mkdir -p $(build_path)/.gcloud + ensure-kube-dirs: -mkdir -p ~/.config/helm - -mkdir -p ~/.kube \ No newline at end of file + -mkdir -p ~/.kube + +ci-gcloud-auth-cluster: + # Internal CI target. Used to authenticate against the integration test cluster. + docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) --network=cloudbuild \ + -e "USE_GKE_GCLOUD_AUTH_PLUGIN=True" --entrypoint=gcloud $(BUILD_IMAGE_TAG) container clusters get-credentials agones --zone us-west1-c diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 9ea2d41aba..43a831e2df 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,35 +16,34 @@ steps: - name: gcr.io/cloud-builders/git args: [ submodule, update, --init, --recursive ] id: fetch-git-submodules + waitFor: + - "-" - name: gcr.io/cloud-builders/docker args: [ pull, "${_BUILD_IMAGE_TAG}" ] id: pull-build-image + waitFor: + - "-" - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker dir: ./build args: - - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG} - - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG} - test-quilkin id: test-quilkin + waitFor: + - fetch-git-submodules + - pull-build-image - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker dir: ./build args: - - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG} - - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG} - test-examples id: test-examples - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker dir: ./build args: - - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG} - - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG} - test-docs id: test-docs - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker dir: ./build args: - - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG} - - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG} - build id: build # Run the built images for 5 seconds to make sure that the entrypoint and default config works out of the box @@ -53,21 +52,47 @@ steps: entrypoint: bash args: - '-c' - - 'timeout --signal=INT --preserve-status 5s docker run --rm quilkin:$(make version)' + - 'timeout --signal=INT --preserve-status 5s docker run --rm ${_REPOSITORY}quilkin:$(make version)' id: test-quilkin-debug + waitFor: + - build - name: gcr.io/cloud-builders/docker dir: ./build entrypoint: bash args: - '-c' - - 'timeout --signal=INT --preserve-status 5s docker run --rm quilkin:$(make version)-debug' + - 'timeout --signal=INT --preserve-status 5s docker run --rm ${_REPOSITORY}quilkin:$(make version)-debug' id: test-quilkin-release + waitFor: + - build + - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker + dir: ./build + args: + - ci-gcloud-auth-cluster + id: gcloud-auth-cluster + waitFor: + - test-quilkin + - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker + dir: ./build + args: + - SKIP_BUILD_IMAGE=1 + - DELETE_DELAY_SECONDS=3600 + - DOCKER_RUN_ARGS=--network=cloudbuild + - test-agones + id: test-agones + waitFor: + - gcloud-auth-cluster + - build options: env: - "CARGO_HOME=/workspace/.cargo" + - "REPOSITORY=${_REPOSITORY}" + - "BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}" + - "BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}" machineType: E2_HIGHCPU_32 dynamic_substitutions: true timeout: 7200s substitutions: _BUILD_IMAGE_TAG: us-docker.pkg.dev/${PROJECT_ID}/ci/build-image + _REPOSITORY: us-docker.pkg.dev/${PROJECT_ID}/ci/ logsBucket: "gs://quilkin-build-logs" From 29c00f070f25d54cb39198d3b0b7d93bc56729ab Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Thu, 28 Jul 2022 15:38:18 -0700 Subject: [PATCH 2/3] Review updates. --- agones/src/lib.rs | 8 +++++--- cloudbuild.yaml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/agones/src/lib.rs b/agones/src/lib.rs index 73081e3d49..c4bc5beee0 100644 --- a/agones/src/lib.rs +++ b/agones/src/lib.rs @@ -62,7 +62,7 @@ impl Client { Client { kubernetes: client.clone(), namespace: setup_namespace(client).await, - quilkin_image: env::var_os(IMAGE_TAG).unwrap().into_string().unwrap(), + quilkin_image: env::var(IMAGE_TAG).unwrap(), } }) .await @@ -79,8 +79,10 @@ async fn setup_namespace(client: kube::Client) -> String { let nss = namespaces.list(&lp).await.unwrap(); let dp = DeleteParams::default(); - let delay = env::var_os(DELETE_DELAY_SECONDS) - .map(|value| chrono::Duration::seconds(value.into_string().unwrap().parse().unwrap())); + let delay = env::var(DELETE_DELAY_SECONDS) + .ok() + .and_then(|s| s.parse::().ok()) + .map(chrono::Duration::seconds); for ns in nss { let name = ns.name(); diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 43a831e2df..22beab29b2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,7 +22,7 @@ steps: args: [ pull, "${_BUILD_IMAGE_TAG}" ] id: pull-build-image waitFor: - - "-" + - "-" # Run immediately, don't wait for any previous steps - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker dir: ./build args: From 0ee70ae8c5a323364af0629a3075ec8701362113 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Thu, 28 Jul 2022 19:43:52 -0700 Subject: [PATCH 3/3] Fix issue with long build times. --- build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Makefile b/build/Makefile index a5d7188992..2185e72544 100644 --- a/build/Makefile +++ b/build/Makefile @@ -24,7 +24,7 @@ build_path := $(dir $(mkfile_path)) project_path := $(realpath $(build_path)/..) CARGO_HOME ?= ~/.cargo -BUILD_IMAGE_TAG := quilkin-build +BUILD_IMAGE_TAG ?= quilkin-build rust_toolchain := $(shell grep channel $(project_path)/rust-toolchain.toml | awk '{ print $$3 }') # if this is a release, don't put the sha, otherwise, leave it off.