Run Agones tests in CI

This makes it such that on each PR, the Agones integration tests are run
against the GKE + Agones cluster specified in ./agones/main.tf

This includes:
* Fix for main.tf to align release branch with Agones version.
* Configure how long to wait before cleaning up past test namespaces.
* Cleanup and optimisation of cloudbuild.yaml.

Work on googleforgames#510

agones/main.tf

@@ -68,7 +68,7 @@ module "agones_cluster" {
 module "helm_agones" {
   source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/helm3/?ref=release-1.24.0"
 
-  agones_version         = "1.23.0"
+  agones_version         = "1.24.0"
   values_file            = ""
   chart                  = "agones"
   host                   = module.agones_cluster.host

agones/src/lib.rs

@@ -20,7 +20,9 @@ use std::{
     time::{SystemTime, UNIX_EPOCH},
 };
 
-use k8s_openapi::{api::core::v1::Namespace, apimachinery::pkg::apis::meta::v1::ObjectMeta};
+use k8s_openapi::{
+    api::core::v1::Namespace, apimachinery::pkg::apis::meta::v1::ObjectMeta, chrono,
+};
 use kube::{
     api::{DeleteParams, ListParams, PostParams},
     Api, ResourceExt,
@@ -33,6 +35,7 @@ mod pod;
 static CLIENT: OnceCell<Client> = OnceCell::const_new();
 #[allow(dead_code)]
 const IMAGE_TAG: &str = "IMAGE_TAG";
+const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS";
 
 pub struct Client {
     /// The Kubernetes client
@@ -75,10 +78,23 @@ async fn setup_namespace(client: kube::Client) -> String {
     let lp = ListParams::default().labels("owner=quilkin-test");
     let nss = namespaces.list(&lp).await.unwrap();
     let dp = DeleteParams::default();
+
+    let delay = env::var_os(DELETE_DELAY_SECONDS)
+        .map(|value| chrono::Duration::seconds(value.into_string().unwrap().parse().unwrap()));
+
     for ns in nss {
         let name = ns.name();
-        if let Err(err) = namespaces.delete(name.as_str(), &dp).await {
-            println!("Failure attempting to deleted namespace: {:?}, {err}", name);
+
+        let delete = delay
+            .and_then(|duration| {
+                let expiry = ns.creation_timestamp()?.0 + duration;
+                Some(chrono::Utc::now() > expiry)
+            })
+            .unwrap_or(true);
+        if delete {
+            if let Err(err) = namespaces.delete(name.as_str(), &dp).await {
+                println!("Failure attempting to deleted namespace: {:?}, {err}", name);
+            }
         }
     }
 

build/Makefile

@@ -49,8 +49,8 @@ helm_config := ~/.config/helm
 helm_cache := ~/.cache/helm
 
 kube_mount_args := -v $(kubeconfig_path):/root/.kube -v $(helm_config):/root/.config/helm -v $(helm_cache):/root/.cache/helm
-
 minikube_args := --network=host -v ~/.minikube:$(HOME)/.minikube
+gcloud_mount_args := -v $(build_path)/.config/gcloud:/root/.config/gcloud
 
 #   _____                    _
 #  |_   _|_ _ _ __ __ _  ___| |_ ___
@@ -165,8 +165,12 @@ run-test-agones:
 
 # Convenience target to build and push quilkin images to a repository.
 # Use `REPOSITORY` arg to specify the repository to push to.
+# USe `SKIP_BUILD_IMAGE` if you want to skip building the image if it has been already built.
 # See `build-image` for more details.
+push:
+ifndef SKIP_BUILD_IMAGE
 push: build-image
+endif
 	docker push $(IMAGE_TAG)-debug
 	docker push $(IMAGE_TAG)
 
@@ -196,10 +200,9 @@ docs: ensure-build-image
 
 # Start an interactive shell inside the build image
 # Useful for testing, or adhoc cargo, gcloud, kubectl or terraform commands
-shell: ensure-kube-dirs ensure-build-image
-	-mkdir -p $(build_path)/.gcloud
+shell: ensure-gcloud-dirs ensure-kube-dirs ensure-build-image
 	docker run --rm -it $(DOCKER_RUN_ARGS) $(common_rust_args) \
-		-v $(build_path)/.config/gcloud:/root/.config/gcloud $(kube_mount_args) \
+		$(gcloud_mount_args) $(kube_mount_args) \
 		 --entrypoint=bash $(BUILD_IMAGE_TAG)
 
 ensure-build-image: ensure-cargo-registry
@@ -208,6 +211,14 @@ ensure-build-image: ensure-cargo-registry
 ensure-cargo-registry:
 	-mkdir -p $(CARGO_HOME)/registry
 
+ensure-gcloud-dirs:
+	-mkdir -p $(build_path)/.gcloud
+
 ensure-kube-dirs:
 	-mkdir -p ~/.config/helm
-	-mkdir -p ~/.kube
+	-mkdir -p ~/.kube
+
+ci-gcloud-auth-cluster:
+	# Internal CI target. Used to authenticate against the integration test cluster.
+	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) --network=cloudbuild \
+		   -e "USE_GKE_GCLOUD_AUTH_PLUGIN=True" --entrypoint=gcloud $(BUILD_IMAGE_TAG) container clusters get-credentials agones --zone us-west1-c

cloudbuild.yaml

@@ -16,35 +16,34 @@ steps:
   - name: gcr.io/cloud-builders/git
     args: [ submodule, update, --init, --recursive ]
     id: fetch-git-submodules
+    waitFor:
+      - "-"
   - name: gcr.io/cloud-builders/docker
     args: [ pull, "${_BUILD_IMAGE_TAG}" ]
     id: pull-build-image
+    waitFor:
+      - "-"
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
     dir: ./build
     args:
-      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
-      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
       - test-quilkin
     id: test-quilkin
+    waitFor:
+      - fetch-git-submodules
+      - pull-build-image
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
     dir: ./build
     args:
-      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
-      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
       - test-examples
     id: test-examples
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
     dir: ./build
     args:
-      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
-      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
       - test-docs
     id: test-docs
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
     dir: ./build
     args:
-      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
-      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
       - build
     id: build
   # Run the built images for 5 seconds to make sure that the entrypoint and default config works out of the box
@@ -53,21 +52,46 @@ steps:
     entrypoint: bash
     args:
       - '-c'
-      - 'timeout --signal=INT --preserve-status 5s docker run --rm quilkin:$(make version)'
+      - 'timeout --signal=INT --preserve-status 5s docker run --rm ${_REPOSITORY}quilkin:$(make version)'
     id: test-quilkin-debug
+    waitFor:
+      - build
   - name: gcr.io/cloud-builders/docker
     dir: ./build
     entrypoint: bash
     args:
       - '-c'
-      - 'timeout --signal=INT --preserve-status 5s docker run --rm quilkin:$(make version)-debug'
+      - 'timeout --signal=INT --preserve-status 5s docker run --rm ${_REPOSITORY}quilkin:$(make version)-debug'
     id: test-quilkin-release
+    waitFor:
+      - build
+  - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
+    dir: ./build
+    args:
+      - ci-gcloud-auth-cluster
+    id: gcloud-auth-cluster
+    waitFor:
+      - test-quilkin
+  - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
+    dir: ./build
+    args:
+      - SKIP_BUILD_IMAGE=1
+      - DOCKER_RUN_ARGS=--network=cloudbuild
+      - test-agones
+    id: test-agones
+    waitFor:
+      - gcloud-auth-cluster
+      - build
 options:
   env:
     - "CARGO_HOME=/workspace/.cargo"
+    - "REPOSITORY=${_REPOSITORY}"
+    - "BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}"
+    - "BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}"
   machineType: E2_HIGHCPU_32
   dynamic_substitutions: true
 timeout: 7200s
 substitutions:
   _BUILD_IMAGE_TAG: us-docker.pkg.dev/${PROJECT_ID}/ci/build-image
+  _REPOSITORY: us-docker.pkg.dev/${PROJECT_ID}/ci/
 logsBucket: "gs://quilkin-build-logs"