Dev: Build, Push and Test Agones Integrations

.dockerignore

@@ -14,14 +14,39 @@
 # limitations under the License.
 #
 
+*.TAG
+*.hbs
+*.md
+*.png
+*.sh
+*.toml
+*.yaml
+*.zip
+*.iml
 .*
-target/*/deps
-target/*/incremental
-target/*/build
+/agones
+/benches
+/build
+/docs
+/examples
+/macros
+/proto
+/sdks
+/src
+/tests
+target/debug
+target/release
+target/**.d
+target/*/*/build
 target/*/*/deps
 target/*/*/incremental
-target/*/*/build
+target/criterion
+target/doc
+target/tmp
 
 # windows files
 *.exe
-*.rlib
+*.rlib
+
+!dependencies-src.zip
+!image/quilkin.yaml

about.toml

@@ -19,5 +19,5 @@ accepted = [
     "BSD-2-Clause",
     "BSD-3-Clause",
     "MIT",
-    "Unicode-DFS-2016",
+    "Unicode-DFS-2016"
 ]

agones/README.md

@@ -7,17 +7,6 @@ This folder containers the integration tests for Quilkin and Agones integration.
 * A Kubernetes cluster with [Agones](https://agones.dev) installed.
 * Local authentication to the cluster via `kubectl`.
 
-## Running Tests
-
-To run the tests, run `cargo test` in this folder. This will run the e2e to tests with the default Quilkin image.
-
-When writing new tests for new features, you will want to specify a development image hosted on a container 
-registry to test against. This can be done through the `QUILKIN_IMAGE` environment variable like so:
-
-```shell
-QUILKIN_IMAGE=us-docker.pkg.dev/my-project-name/dev/quilkin:0.4.0-dev cargo test
-```
-
 ## Creating an Agones Minikube Cluster
 
 If you want to test locally, you can use a tool such a [minikube](https://github.com/kubernetes/minikube) to create 
@@ -55,6 +44,68 @@ terraform apply -var project="<YOUR_GCP_ProjectID>"
 gcloud container clusters get-credentials --zone us-west1-c agones
 ```
 
+## Running Tests
+
+To run the Agones integration tests with a Quilkin image, you will need to specify the image 
+to be used along with the `cargo test` command and ensure that is available on the currently authenticated Kubernetes 
+cluster.
+
+This can be done through the `IMAGE_TAG` environment variable like so:
+
+```shell
+IMAGE_TAG=us-docker.pkg.dev/my-project-name/dev/quilkin:0.4.0-auyz cargo test
+```
+
+### Build, Push and Test in one Go 💪
+
+The [`build/Makefile`](../build/Makefile) provides a targets that can be executed to build a development image, 
+push it a appropriate location, and run the set of Agones integration tests, depending on where you Kubernetes 
+cluster is set up. 
+
+#### Minikube
+
+This target assumes that you have a [working minikube cluster](#creating-an-agones-minikube-cluster),
+under the profile `quilkin`, with Agones installed, and the local `.kube` configuration is currently
+authenticated against it.
+
+To build, push and run the tests:
+
+```shell
+make minikube-test-agones
+```
+
+To change from the default profile of `quilkin`, use the variable `MINIKUBE_PROFILE` to do so.
+
+To pass extra arguments to `cargo test`, to run only a single test, for example, use the `ARGS` variable
+to pass through those options.
+
+#### Hosted Kubernetes Cluster
+
+This target assumes that you have a
+[working hosted Kubernetes cluster, such as GKE](#creating-an-agones-gke-cluster-with-terraform),
+with Agones installed, the local `.kube` configuration is currently authenticated against it,
+and a hosted [docker repository](https://docs.docker.com/docker-hub/repos/) such as
+[Artifact Registry](https://cloud.google.com/artifact-registry) has been provisioned.
+
+To build, push and run the tests:
+
+```shell
+REPOSITORY=us-docker.pkg.dev/my-project/repository-name/ make test-agones
+```
+
+Where `REPOSITORY` is the provisioned Docker repository to push the development image to, and utilise in the 
+integration tests.
+
+> Note: The REPOSITORY variable will need to end with a trailing slash: /.
+
+To pass extra arguments to `cargo test`, to run only a single test, for example, use the `ARGS` variable
+to pass through those options.
+
+### Troubleshooting
+
+If you ever have authentication issues sending commands to the cluster from the e2e test, run a `kubectl` 
+command (e.g. `kubectl get pods`) against the designated cluster to refresh the authentication token and try again. 
+
 ## Licence
 
 Apache 2.0

agones/main.tf

@@ -54,7 +54,7 @@ variable "node_count" {
 
 // Create a GKE cluster with the appropriate structure
 module "agones_cluster" {
-  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=release-1.23.0"
+  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=release-1.24.0"
 
   cluster = {
     "name"             = var.name
@@ -66,7 +66,7 @@ module "agones_cluster" {
 
 // Install Agones via Helm
 module "helm_agones" {
-  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/helm3/?ref=release-1.23.0"
+  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/helm3/?ref=release-1.24.0"
 
   agones_version         = "1.23.0"
   values_file            = ""

agones/src/lib.rs

@@ -32,7 +32,7 @@ mod pod;
 #[allow(dead_code)]
 static CLIENT: OnceCell<Client> = OnceCell::const_new();
 #[allow(dead_code)]
-const QUILKIN_IMAGE: &str = "QUILKIN_IMAGE";
+const IMAGE_TAG: &str = "IMAGE_TAG";
 
 pub struct Client {
     /// The Kubernetes client
@@ -48,7 +48,7 @@ impl Client {
     /// Executes the setup required:
     /// * Creates a test namespace for this test
     /// * Removes previous test namespaces
-    /// * Retrieves the QUILKIN_IMAGE to test from env vars, and defaults it if not available.
+    /// * Retrieves the IMAGE_TAG to test from env vars, and panics if it if not available.
     pub async fn new() -> &'static Client {
         CLIENT
             .get_or_init(|| async {
@@ -59,10 +59,7 @@ impl Client {
                 Client {
                     kubernetes: client.clone(),
                     namespace: setup_namespace(client).await,
-                    quilkin_image: env::var_os(QUILKIN_IMAGE)
-                        .unwrap_or_else(|| "us-docker.pkg.dev/quilkin/release/quilkin:0.3.0".into())
-                        .into_string()
-                        .unwrap(),
+                    quilkin_image: env::var_os(IMAGE_TAG).unwrap().into_string().unwrap(),
                 }
             })
             .await

build/Makefile

@@ -26,7 +26,19 @@ project_path := $(realpath $(build_path)/..)
 CARGO_HOME ?= ~/.cargo
 BUILD_IMAGE_TAG := quilkin-build
 rust_toolchain := $(shell grep channel $(project_path)/rust-toolchain.toml | awk '{ print $$3 }')
-package_version := $(shell grep version -m 1 $(project_path)/Cargo.toml | awk '{print $$3}')
+
+# if this is a release, don't put the sha, otherwise, leave it off.
+ifdef QUILKIN_RELEASE
+	package_version := $(shell grep version -m 1 $(project_path)/Cargo.toml | awk '{print $$3}')
+else
+	git_sha := $(shell git rev-parse --short=7 HEAD)
+	package_version := $(shell grep version -m 1 $(project_path)/Cargo.toml | awk '{print $$3}')-${git_sha}
+endif
+
+# Set this value if you want to use an external registry
+REPOSITORY ?= ""
+IMAGE_TAG ?= ${REPOSITORY}quilkin:$(package_version)
+MINIKUBE_PROFILE ?= quilkin
 
 common_rust_args := -v $(project_path):/workspace -w /workspace \
 					-v $(CARGO_HOME)/registry:/usr/local/cargo/registry
@@ -38,6 +50,8 @@ helm_cache := ~/.cache/helm
 
 kube_mount_args := -v $(kubeconfig_path):/root/.kube -v $(helm_config):/root/.config/helm -v $(helm_cache):/root/.cache/helm
 
+minikube_args := --network=host -v ~/.minikube:$(HOME)/.minikube
+
 #   _____                    _
 #  |_   _|_ _ _ __ __ _  ___| |_ ___
 #    | |/ _` | '__/ _` |/ _ \ __/ __|
@@ -84,19 +98,6 @@ test-docs: ensure-build-image
 			rm /tmp/docs/book/print.html && \
 			htmltest -c /workspace/docs/htmltest.yaml /tmp/docs'
 
-# Run e2e tests against an Agones Kubernetes cluster
-# Not part of `test` as it requires a Kubernetes cluster to be provisioned and running.
-# Use QUILKIN_IMAGE to specify which container image to use in the tests
-test-agones: ensure-kube-dirs ensure-build-image
-test-agones: QUILKIN_IMAGE ?= us-docker.pkg.dev/quilkin/release/quilkin:0.3.0
-test-agones:
-	docker run --rm $(common_rust_args) -w /workspace/agones  \
-			--entrypoint=cargo $(BUILD_IMAGE_TAG) clippy --tests -- -D warnings
-	docker run --rm $(common_rust_args) -w /workspace/agones \
-		--entrypoint=cargo $(BUILD_IMAGE_TAG) fmt -- --check
-	docker run --rm $(common_rust_args) $(kube_mount_args) -w /workspace/agones -e "QUILKIN_IMAGE=${QUILKIN_IMAGE}" \
-			--entrypoint=cargo $(BUILD_IMAGE_TAG) test
-
 # Build all binaries, images and related artifacts
 build: binary-archive build-image
 
@@ -130,16 +131,57 @@ build-macos-binary:
 			sh -c "rustup target add x86_64-apple-darwin && cargo build --target x86_64-apple-darwin && cargo build --release --target x86_64-apple-darwin"
 
 # Build release and debug container images.
-# Use IMAGE_NAME argument to specify the container registry and image name. Defaults to 'quilkin'.
+# Use either `REPOSITORY` to specify a container repository (defaults to blank/none), or use `IMAGE_TAG` argument to specify
+# the entire image name and tag. Defaults to `quilkin:${version}-${git-sha}`.
 build-image: ensure-build-image build-linux-binary
-build-image: IMAGE_NAME := quilkin
 build-image:
 	docker run --rm $(common_rust_args) \
 		--entrypoint=bash $(BUILD_IMAGE_TAG) -c 'cargo about generate license.html.hbs > license.html'
 	docker run --rm $(common_rust_args) \
 		--entrypoint=bash $(BUILD_IMAGE_TAG) -c './image/archive_dependencies.sh'
-	docker build -t $(IMAGE_NAME):$(package_version)-debug --build-arg PROFILE=debug -f $(project_path)/image/Dockerfile $(project_path)
-	docker build -t $(IMAGE_NAME):$(package_version) --build-arg PROFILE=release -f $(project_path)/image/Dockerfile $(project_path)
+	docker build -t $(IMAGE_TAG)-debug --build-arg PROFILE=debug -f $(project_path)/image/Dockerfile $(project_path)
+	docker build -t $(IMAGE_TAG) --build-arg PROFILE=release -f $(project_path)/image/Dockerfile $(project_path)
+
+# Builds Quilkin, pushes it to a repository (use REPOSITORY arg to set value)
+# and then runs the Agones integration tests. See targets `build-images` and `push` for more options and details.
+# Not part of `test` as it requires a Kubernetes cluster to be provisioned and running.
+# To pass extra arguments to `cargo test`, to run only a single test, for example, use the `ARGS` variable
+# to set those options.
+test-agones: push
+test-agones:
+	$(MAKE) run-test-agones
+
+run-test-agones: ensure-kube-dirs
+run-test-agones:
+	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) -w /workspace/agones  \
+			--entrypoint=cargo $(BUILD_IMAGE_TAG) clippy --tests -- -D warnings
+	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) -w /workspace/agones \
+		--entrypoint=cargo $(BUILD_IMAGE_TAG) fmt -- --check
+	# Confirm access to K8s, and force refresh of the auth token for the kube context
+	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \
+		--entrypoint=kubectl $(BUILD_IMAGE_TAG) get ns
+	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \
+			-e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS)
+
+# Convenience target to build and push quilkin images to a repository.
+# Use `REPOSITORY` arg to specify the repository to push to.
+# See `build-image` for more details.
+push: build-image
+	docker push $(IMAGE_TAG)-debug
+	docker push $(IMAGE_TAG)
+
+# Convenience target to build and push quilkin images into a minikube instance
+# Use `MINIKUBE_PROFILE` to specify the profile. Defaults to `quilkin`.
+minikube-push: build-image
+	minikube image load $(IMAGE_TAG) -p $(MINIKUBE_PROFILE)
+
+# Builds Quilkin, pushes it to a minikube instance (use `MINIKUBE_PROFILE` to specify the profile. Defaults to `quilkin`)
+# and then runs the Agones integration tests. See targets `build-images` and `minikube-push` for more options and details.
+# Not part of `test` as it requires a Kubernetes cluster to be provisioned and running.
+# To pass extra arguments to `cargo test`, to run only a single test, for example, use the `ARGS` variable
+# to set those options.
+minikube-test-agones: minikube-push
+	$(MAKE) DOCKER_RUN_ARGS="$(minikube_args)" run-test-agones
 
 # Runs mdbook and cargo doc in the same directory structure as what is hosted on Github pages.
 # Open http://localhost:3000/book/index.html or http://localhost:3000/api/quilkin/index.html after running. Pages will live reload on change.
@@ -156,7 +198,7 @@ docs: ensure-build-image
 # Useful for testing, or adhoc cargo, gcloud, kubectl or terraform commands
 shell: ensure-kube-dirs ensure-build-image
 	-mkdir -p $(build_path)/.gcloud
-	docker run --rm -it $(common_rust_args) \
+	docker run --rm -it $(DOCKER_RUN_ARGS) $(common_rust_args) \
 		-v $(build_path)/.config/gcloud:/root/.config/gcloud $(kube_mount_args) \
 		 --entrypoint=bash $(BUILD_IMAGE_TAG)
 

build/README.md

@@ -53,6 +53,8 @@ To test dependency licences and security advisories:
 
 `cargo deny check`
 
+See the [agones](../agones) folder for the [Agones](https://agones.dev) integration testing tooling.
+
 ### Developing with Make + Docker 
 
 There are a few reasons you may want to use the [Make](https://www.gnu.org/software/make/)
@@ -70,11 +72,14 @@ To use the tooling for Make + Docker testing and development, you will need:
 
 #### Run tests
 
-`make test` will run all tests for this project.
+`make test` will run all tests for this project, except the [Agones](https:/agones.dev) integration tests.
 
 This can be useful if continuous integration is failing, but you are unable to reproduce the issue using 
 your local Rust tooling
 
+See the [agones](../agones) folder for the `make` targets to build, push and run the [Agones](https://agones.dev) 
+integration testing tooling.
+
 #### Build everything
 
 To build all operating system binaries, as well as the container image for your current development version of 

build/release/cloudbuild.yaml

@@ -30,7 +30,7 @@ steps:
     args:
       - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
       - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
-      - IMAGE_NAME=${_IMAGE_TAG}
+      - REPOSITORY=${_REPOSITORY}
       - build
     id: build
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
@@ -40,14 +40,17 @@ steps:
       - '-c'
       - 'make version > ../version'
     id: output-version
-  - name: ferrarimarco/github-changelog-generator:1.15.2
+    waitFor: ['-']
+  - name: githubchangeloggenerator/github-changelog-generator:latest # have to use latest, there is no 1.16.4 tag
     id: changelog
     entrypoint: sh
     args:
       - '-c'
       - 'github_changelog_generator --user=googleforgames --project=quilkin --bug-labels=kind/bug --enhancement-labels=kind/feature --breaking-labels=kind/breaking --security-labels=area/security --exclude-labels=duplicate,question,invalid,wontfix,priority/wontfix --future-release "v$(cat ./version)" --release-branch=main --token $$GH_TOKEN'
     secretEnv:
       - GH_TOKEN
+    waitFor:
+      - output-version
 artifacts:
   objects:
     location: '$_STORAGE'
@@ -56,16 +59,17 @@ artifacts:
       - 'license.html'
       - 'CHANGELOG.md'
   images:
-    - $_IMAGE_TAG
+    - ${_REPOSITORY}quilkin
 options:
   env:
     - "CARGO_HOME=/workspace/.cargo"
     - "RUST_BACKTRACE=1"
+    - "QUILKIN_RELEASE=1"
   machineType: E2_HIGHCPU_32
   dynamic_substitutions: true
 substitutions:
   _BUILD_IMAGE_TAG: us-docker.pkg.dev/${PROJECT_ID}/ci/build-image
-  _IMAGE_TAG: us-docker.pkg.dev/${PROJECT_ID}/release/quilkin
+  _REPOSITORY: us-docker.pkg.dev/${PROJECT_ID}/release/
   _STORAGE: gs://${PROJECT_ID}-quilkin-releases
 availableSecrets:
   secretManager: