From 50aa4278bf4d85f63b70288a504f4ae457bed6db Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Mon, 29 Aug 2022 13:11:42 -0700 Subject: [PATCH] Agones GameServer + Quilkin sidecar test Implements an integration test wherein we use Quilkin as a sidecar to an Agones GameServer container. Also includes some updates to the test harnesses: * Function to create a common Quilkin container * Fixed odd (?) bug where k8s client would sometimes close at the end of tests, failing subsequent other tests. * Convenience function for grabbing the address of a GameServer so you can send packets to it. Work on #510 --- agones/src/lib.rs | 62 +++++++++++++++++++++++-- agones/src/pod.rs | 10 ++-- agones/src/sidecar.rs | 105 ++++++++++++++++++++++++++++++++++++++---- build/Makefile | 2 +- 4 files changed, 157 insertions(+), 22 deletions(-) diff --git a/agones/src/lib.rs b/agones/src/lib.rs index 25e4fcd945..781126bf83 100644 --- a/agones/src/lib.rs +++ b/agones/src/lib.rs @@ -23,11 +23,14 @@ use std::{ use k8s_openapi::{ api::{ core::v1::{ - Container, Namespace, PodSpec, PodTemplateSpec, ResourceRequirements, ServiceAccount, + Container, HTTPGetAction, Namespace, PodSpec, PodTemplateSpec, Probe, + ResourceRequirements, ServiceAccount, VolumeMount, }, rbac::v1::{RoleBinding, RoleRef, Subject}, }, - apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::ObjectMeta}, + apimachinery::pkg::{ + api::resource::Quantity, apis::meta::v1::ObjectMeta, util::intstr::IntOrString, + }, chrono, }; use kube::{ @@ -54,6 +57,7 @@ const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS"; /// for more details. pub const GAMESERVER_IMAGE: &str = "gcr.io/agones-images/simple-game-server:0.13"; +#[derive(Clone)] pub struct Client { /// The Kubernetes client pub kubernetes: kube::Client, @@ -64,13 +68,13 @@ pub struct Client { } impl Client { - /// Thread safe way to create a Client once and only once across multiple tests. + /// Thread safe way to create a Clients across multiple tests. /// Executes the setup required: /// * Creates a test namespace for this test /// * Removes previous test namespaces /// * Retrieves the IMAGE_TAG to test from env vars, and panics if it if not available. - pub async fn new() -> &'static Client { - CLIENT + pub async fn new() -> Client { + let mut client = CLIENT .get_or_init(|| async { let client = kube::Client::try_default() .await @@ -83,6 +87,14 @@ impl Client { } }) .await + .clone(); + + // create a new client on each invocation, as the client can close + // at the end of each test. + client.kubernetes = kube::Client::try_default() + .await + .expect("Kubernetes client to be created"); + client } } @@ -240,3 +252,43 @@ pub fn is_gameserver_ready() -> impl Condition { .unwrap_or(false) } } + +/// Returns a container for Quilkin, with an optional volume mount name +pub fn quilkin_container(client: &Client, volume_mount: Option) -> Container { + let mut container = Container { + name: "quilkin".into(), + image: Some(client.quilkin_image.clone()), + liveness_probe: Some(Probe { + http_get: Some(HTTPGetAction { + path: Some("/live".into()), + port: IntOrString::Int(9091), + ..Default::default() + }), + initial_delay_seconds: Some(3), + period_seconds: Some(2), + ..Default::default() + }), + ..Default::default() + }; + + if let Some(name) = volume_mount { + container.volume_mounts = Some(vec![VolumeMount { + name, + mount_path: "/etc/quilkin".into(), + ..Default::default() + }]) + }; + + container +} + +/// Convenience function to return the address with the first port of GameServer +pub fn gameserver_address(gs: &GameServer) -> String { + let status = gs.status.as_ref().unwrap(); + let address = format!( + "{}:{}", + status.address, + status.ports.as_ref().unwrap()[0].port + ); + address +} diff --git a/agones/src/pod.rs b/agones/src/pod.rs index a4e571490b..b2d4212d80 100644 --- a/agones/src/pod.rs +++ b/agones/src/pod.rs @@ -17,7 +17,7 @@ #[cfg(test)] mod tests { use k8s_openapi::{ - api::core::v1::{Container, Pod, PodSpec}, + api::core::v1::{Pod, PodSpec}, apimachinery::pkg::apis::meta::v1::ObjectMeta, }; use kube::{ @@ -28,7 +28,7 @@ mod tests { use std::time::Duration; use tokio::time::timeout; - use crate::Client; + use crate::{quilkin_container, Client}; #[tokio::test] async fn create_quilkin_pod() { @@ -41,11 +41,7 @@ mod tests { ..Default::default() }, spec: Some(PodSpec { - containers: vec![Container { - name: "quilkin".into(), - image: Some(client.quilkin_image.clone()), - ..Default::default() - }], + containers: vec![quilkin_container(&client, None)], ..Default::default() }), status: None, diff --git a/agones/src/sidecar.rs b/agones/src/sidecar.rs index abbcc0cd3e..dd340a6fc8 100644 --- a/agones/src/sidecar.rs +++ b/agones/src/sidecar.rs @@ -16,13 +16,14 @@ #[cfg(test)] mod tests { - use crate::{game_server, is_gameserver_ready, Client}; - use kube::api::PostParams; - use kube::runtime::wait::await_condition; - use kube::{Api, ResourceExt}; - use quilkin::config::watch::agones::crd::GameServer; - use quilkin::test_utils::TestHelper; - use std::time::Duration; + use crate::{game_server, is_gameserver_ready, quilkin_container, Client}; + use k8s_openapi::{ + api::core::v1::{ConfigMap, ConfigMapVolumeSource, Volume}, + apimachinery::pkg::apis::meta::v1::ObjectMeta, + }; + use kube::{api::PostParams, runtime::wait::await_condition, Api, ResourceExt}; + use quilkin::{config::watch::agones::crd::GameServer, test_utils::TestHelper}; + use std::{collections::BTreeMap, time::Duration}; use tokio::time::timeout; #[tokio::test] @@ -49,8 +50,7 @@ mod tests { let t = TestHelper::default(); let recv = t.open_socket_and_recv_single_packet().await; - let status = gs.status.unwrap(); - let address = format!("{}:{}", status.address, status.ports.unwrap()[0].port); + let address = crate::gameserver_address(&gs); recv.socket .send_to("hello".as_bytes(), address) .await @@ -62,4 +62,91 @@ mod tests { .unwrap(); assert_eq!("ACK: hello\n", response); } + + #[tokio::test] + /// Testing Quilkin running as a sidecar next to a GameServer + async fn gameserver_sidecar() { + let client = Client::new().await; + let config_maps: Api = + Api::namespaced(client.kubernetes.clone(), client.namespace.as_str()); + let gameservers: Api = + Api::namespaced(client.kubernetes.clone(), client.namespace.as_str()); + let pp = PostParams::default(); + + // We'll append "sidecar", to prove the packet goes through the sidecar. + let config = r#" +version: v1alpha1 +filters: + - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + config: + on_read: APPEND + on_write: DO_NOTHING + bytes: c2lkZWNhcg== # sidecar +clusters: + default: + localities: + - endpoints: + - address: 127.0.0.1:7654 +"#; + + let config_map = ConfigMap { + metadata: ObjectMeta { + generate_name: Some("quilkin-config-".into()), + ..Default::default() + }, + data: Some(BTreeMap::from([( + "quilkin.yaml".to_string(), + config.to_string(), + )])), + ..Default::default() + }; + + let config_map = config_maps.create(&pp, &config_map).await.unwrap(); + let mut gs = game_server(); + + // reset ports to point at the Quilkin sidecar + gs.spec.ports[0].container_port = 7000; + gs.spec.ports[0].container = Some("quilkin".into()); + + // set the gameserver container to the simple-game-server container. + let mut template = gs.spec.template.spec.as_mut().unwrap(); + gs.spec.container = template.containers[0].name.clone(); + + let mount_name = "config".to_string(); + template + .containers + .push(quilkin_container(&client, Some(mount_name.clone()))); + + template.volumes = Some(vec![Volume { + name: mount_name, + config_map: Some(ConfigMapVolumeSource { + name: Some(config_map.name()), + ..Default::default() + }), + ..Default::default() + }]); + + let gs = gameservers.create(&pp, &gs).await.unwrap(); + let name = gs.name(); + let ready = await_condition(gameservers.clone(), name.as_str(), is_gameserver_ready()); + timeout(Duration::from_secs(30), ready) + .await + .expect("GameServer should be ready") + .unwrap(); + let gs = gameservers.get(name.as_str()).await.unwrap(); + + let t = TestHelper::default(); + let recv = t.open_socket_and_recv_single_packet().await; + let address = crate::gameserver_address(&gs); + recv.socket + .send_to("hello".as_bytes(), address) + .await + .unwrap(); + + let response = timeout(Duration::from_secs(30), recv.packet_rx) + .await + .expect("should receive packet") + .unwrap(); + assert_eq!("ACK: hellosidecar\n", response); + } } diff --git a/build/Makefile b/build/Makefile index b99b86dd08..b9be0a6ac0 100644 --- a/build/Makefile +++ b/build/Makefile @@ -185,7 +185,7 @@ run-test-agones: docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \ --entrypoint=kubectl $(BUILD_IMAGE_TAG) get ns docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \ - -e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS) + -e "RUST_BACKTRACE=1" -e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS) # Convenience target to build and push quilkin images to a repository. # Use `REPOSITORY` arg to specify the repository to push to.