Agones GameServer + Quilkin sidecar test

Implements an integration test wherein we use Quilkin as a sidecar to an
Agones GameServer container.

Also includes some updates to the test harnesses:
* Function to create a common Quilkin container
* Fixed odd (?) bug where k8s client would sometimes close at the end of
  tests, failing subsequent other tests.
* Convenience function for grabbing the address of a GameServer so you
  can send packets to it.

Work on googleforgames#510

agones/src/lib.rs

@@ -23,11 +23,14 @@ use std::{
 use k8s_openapi::{
     api::{
         core::v1::{
-            Container, Namespace, PodSpec, PodTemplateSpec, ResourceRequirements, ServiceAccount,
+            Container, HTTPGetAction, Namespace, PodSpec, PodTemplateSpec, Probe,
+            ResourceRequirements, ServiceAccount, VolumeMount,
         },
         rbac::v1::{RoleBinding, RoleRef, Subject},
     },
-    apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::ObjectMeta},
+    apimachinery::pkg::{
+        api::resource::Quantity, apis::meta::v1::ObjectMeta, util::intstr::IntOrString,
+    },
     chrono,
 };
 use kube::{
@@ -54,6 +57,7 @@ const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS";
 /// for more details.
 pub const GAMESERVER_IMAGE: &str = "gcr.io/agones-images/simple-game-server:0.13";
 
+#[derive(Clone)]
 pub struct Client {
     /// The Kubernetes client
     pub kubernetes: kube::Client,
@@ -64,13 +68,13 @@ pub struct Client {
 }
 
 impl Client {
-    /// Thread safe way to create a Client once and only once across multiple tests.
+    /// Thread safe way to create a Clients across multiple tests.
     /// Executes the setup required:
     /// * Creates a test namespace for this test
     /// * Removes previous test namespaces
     /// * Retrieves the IMAGE_TAG to test from env vars, and panics if it if not available.
-    pub async fn new() -> &'static Client {
-        CLIENT
+    pub async fn new() -> Client {
+        let mut client = CLIENT
             .get_or_init(|| async {
                 let client = kube::Client::try_default()
                     .await
@@ -83,6 +87,14 @@ impl Client {
                 }
             })
             .await
+            .clone();
+
+        // create a new client on each invocation, as the client can close
+        // at the end of each test.
+        client.kubernetes = kube::Client::try_default()
+            .await
+            .expect("Kubernetes client to be created");
+        client
     }
 }
 
@@ -240,3 +252,43 @@ pub fn is_gameserver_ready() -> impl Condition<GameServer> {
             .unwrap_or(false)
     }
 }
+
+/// Returns a container for Quilkin, with an optional volume mount name
+pub fn quilkin_container(client: &Client, volume_mount: Option<String>) -> Container {
+    let mut container = Container {
+        name: "quilkin".into(),
+        image: Some(client.quilkin_image.clone()),
+        liveness_probe: Some(Probe {
+            http_get: Some(HTTPGetAction {
+                path: Some("/live".into()),
+                port: IntOrString::Int(9091),
+                ..Default::default()
+            }),
+            initial_delay_seconds: Some(3),
+            period_seconds: Some(2),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    if let Some(name) = volume_mount {
+        container.volume_mounts = Some(vec![VolumeMount {
+            name,
+            mount_path: "/etc/quilkin".into(),
+            ..Default::default()
+        }])
+    };
+
+    container
+}
+
+/// Convenience function to return the address with the first port of GameServer
+pub fn gameserver_address(gs: &GameServer) -> String {
+    let status = gs.status.as_ref().unwrap();
+    let address = format!(
+        "{}:{}",
+        status.address,
+        status.ports.as_ref().unwrap()[0].port
+    );
+    address
+}

agones/src/pod.rs

@@ -17,7 +17,7 @@
 #[cfg(test)]
 mod tests {
     use k8s_openapi::{
-        api::core::v1::{Container, Pod, PodSpec},
+        api::core::v1::{Pod, PodSpec},
         apimachinery::pkg::apis::meta::v1::ObjectMeta,
     };
     use kube::{
@@ -28,7 +28,7 @@ mod tests {
     use std::time::Duration;
     use tokio::time::timeout;
 
-    use crate::Client;
+    use crate::{quilkin_container, Client};
 
     #[tokio::test]
     async fn create_quilkin_pod() {
@@ -41,11 +41,7 @@ mod tests {
                 ..Default::default()
             },
             spec: Some(PodSpec {
-                containers: vec![Container {
-                    name: "quilkin".into(),
-                    image: Some(client.quilkin_image.clone()),
-                    ..Default::default()
-                }],
+                containers: vec![quilkin_container(&client, None)],
                 ..Default::default()
             }),
             status: None,

agones/src/sidecar.rs

@@ -16,13 +16,14 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::{game_server, is_gameserver_ready, Client};
-    use kube::api::PostParams;
-    use kube::runtime::wait::await_condition;
-    use kube::{Api, ResourceExt};
-    use quilkin::config::watch::agones::crd::GameServer;
-    use quilkin::test_utils::TestHelper;
-    use std::time::Duration;
+    use crate::{game_server, is_gameserver_ready, quilkin_container, Client};
+    use k8s_openapi::{
+        api::core::v1::{ConfigMap, ConfigMapVolumeSource, Volume},
+        apimachinery::pkg::apis::meta::v1::ObjectMeta,
+    };
+    use kube::{api::PostParams, runtime::wait::await_condition, Api, ResourceExt};
+    use quilkin::{config::watch::agones::crd::GameServer, test_utils::TestHelper};
+    use std::{collections::BTreeMap, time::Duration};
     use tokio::time::timeout;
 
     #[tokio::test]
@@ -49,8 +50,7 @@ mod tests {
 
         let t = TestHelper::default();
         let recv = t.open_socket_and_recv_single_packet().await;
-        let status = gs.status.unwrap();
-        let address = format!("{}:{}", status.address, status.ports.unwrap()[0].port);
+        let address = crate::gameserver_address(&gs);
         recv.socket
             .send_to("hello".as_bytes(), address)
             .await
@@ -62,4 +62,91 @@ mod tests {
             .unwrap();
         assert_eq!("ACK: hello\n", response);
     }
+
+    #[tokio::test]
+    /// Testing Quilkin running as a sidecar next to a GameServer
+    async fn gameserver_sidecar() {
+        let client = Client::new().await;
+        let config_maps: Api<ConfigMap> =
+            Api::namespaced(client.kubernetes.clone(), client.namespace.as_str());
+        let gameservers: Api<GameServer> =
+            Api::namespaced(client.kubernetes.clone(), client.namespace.as_str());
+        let pp = PostParams::default();
+
+        // We'll append "sidecar", to prove the packet goes through the sidecar.
+        let config = r#"
+version: v1alpha1
+filters:
+  - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes
+    config:
+        on_read: APPEND
+        on_write: DO_NOTHING
+        bytes: c2lkZWNhcg== # sidecar
+clusters:
+  default:
+    localities:
+      - endpoints:
+          - address: 127.0.0.1:7654
+"#;
+
+        let config_map = ConfigMap {
+            metadata: ObjectMeta {
+                generate_name: Some("quilkin-config-".into()),
+                ..Default::default()
+            },
+            data: Some(BTreeMap::from([(
+                "quilkin.yaml".to_string(),
+                config.to_string(),
+            )])),
+            ..Default::default()
+        };
+
+        let config_map = config_maps.create(&pp, &config_map).await.unwrap();
+        let mut gs = game_server();
+
+        // reset ports to point at the Quilkin sidecar
+        gs.spec.ports[0].container_port = 7000;
+        gs.spec.ports[0].container = Some("quilkin".into());
+
+        // set the gameserver container to the xonotic container.
+        let mut template = gs.spec.template.spec.as_mut().unwrap();
+        gs.spec.container = template.containers[0].name.clone();
+
+        let mount_name = "config".to_string();
+        template
+            .containers
+            .push(quilkin_container(&client, Some(mount_name.clone())));
+
+        template.volumes = Some(vec![Volume {
+            name: mount_name,
+            config_map: Some(ConfigMapVolumeSource {
+                name: Some(config_map.name()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        }]);
+
+        let gs = gameservers.create(&pp, &gs).await.unwrap();
+        let name = gs.name();
+        let ready = await_condition(gameservers.clone(), name.as_str(), is_gameserver_ready());
+        timeout(Duration::from_secs(30), ready)
+            .await
+            .expect("GameServer should be ready")
+            .unwrap();
+        let gs = gameservers.get(name.as_str()).await.unwrap();
+
+        let t = TestHelper::default();
+        let recv = t.open_socket_and_recv_single_packet().await;
+        let address = crate::gameserver_address(&gs);
+        recv.socket
+            .send_to("hello".as_bytes(), address)
+            .await
+            .unwrap();
+
+        let response = timeout(Duration::from_secs(30), recv.packet_rx)
+            .await
+            .expect("should receive packet")
+            .unwrap();
+        assert_eq!("ACK: hellosidecar\n", response);
+    }
 }

build/Makefile

@@ -185,7 +185,7 @@ run-test-agones:
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \
 		--entrypoint=kubectl $(BUILD_IMAGE_TAG) get ns
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/agones \
-			-e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS)
+			-e "RUST_BACKTRACE=1" -e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS)
 
 # Convenience target to build and push quilkin images to a repository.
 # Use `REPOSITORY` arg to specify the repository to push to.