Add overlay tests

tests/integration/overlays/base.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+OUTPUT_FOLDER=generated-cluster-base
+NAMESPACE=default
+
+setup_base() {
+  kubectl apply -f sourcegraph.StorageClass.yaml
+
+  /bin/cat <<EOM >deploy_sourcegraph_git_ssh_config
+Host *
+  StrictHostKeyChecking no
+EOM
+
+  kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret
+}
+
+deploy_base() {
+  mkdir $OUTPUT_FOLDER
+  CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP"
+  cp -r ${DEPLOY_SOURCEGRAPH_ROOT}/base ${CURRENT_DIR}/${OUTPUT_FOLDER}
+
+  GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/base/gitserver/gitserver.StatefulSet.yaml
+  cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS
+  cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS
+
+  kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  # wait for it all to finish (we list out the ones with persistent volume claim because they take longer)
+
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend
+}
+
+cleanup_base() {
+  kubectl delete -n ${NAMESPACE} -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  kubectl delete -f sourcegraph.StorageClass.yaml
+
+  kubectl delete secret -n ${NAMESPACE} gitserver-ssh
+}

tests/integration/overlays/install-src.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ ! "$(which src)" ]; then
+  echo "Installing src to /usr/local/bin"
+  mkdir -p /tmp/src
+  cd /tmp/src
+  wget https://github.com/sourcegraph/src-cli/releases/download/3.21.7/src-cli_3.21.7_linux_amd64.tar.gz
+  tar xvzf src-cli_3.21.7_linux_amd64.tar.gz
+  cp src /usr/local/bin/src
+  chmod a+x /usr/local/bin/src
+fi
+
+if [ ! "$(which jy)" ]; then
+  echo "Installing jy to /usr/local/bin"
+  mkdir -p /tmp/jy
+  cd /tmp/jy
+  wget https://github.com/sourcegraph/jy/releases/download/v1.0.0/jy-1.0.0-linux-amd64
+  cp jy-1.0.0-linux-amd64 /usr/local/bin/jy
+  chmod a+x /usr/local/bin/jy
+fi
+
+if [ ! "$(which yj)" ]; then
+  echo "Installing yj to /usr/local/bin"
+  mkdir -p /tmp/yj
+  cd /tmp/yj
+  wget https://github.com/sourcegraph/yj/releases/download/v1.0.0/yj-1.0.0-linux-amd64
+  cp yj-1.0.0-linux-amd64 /usr/local/bin/yj
+  chmod a+x /usr/local/bin/yj
+fi

tests/integration/overlays/namespaced.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+OUTPUT_FOLDER=generated-cluster-namespaced
+NAMESPACE=ns-sourcegraph
+
+setup_namespaced() {
+  kubectl apply -f sourcegraph.StorageClass.yaml
+
+  kubectl create namespace ${NAMESPACE}
+
+  /bin/cat <<EOM >deploy_sourcegraph_git_ssh_config
+Host *
+  StrictHostKeyChecking no
+EOM
+
+  kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret
+}
+
+deploy_namespaced() {
+  mkdir $OUTPUT_FOLDER
+  CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP"
+  "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh namespaced ${CURRENT_DIR}/${OUTPUT_FOLDER}
+
+  GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml
+  cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS
+  cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS
+
+  kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080
+
+  # wait for it all to finish (we list out the ones with persistent volume claim because they take longer)
+
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend
+}
+
+cleanup_namespaced() {
+  kubectl delete -f sourcegraph.StorageClass.yaml
+
+  kubectl delete namespace ${NAMESPACE}
+}

tests/integration/overlays/nonroot-policy.yaml

@@ -0,0 +1,27 @@
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: nonroot-policy
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  # The rest fills in some required fields.
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  volumes:
+    - '*'
+  readOnlyRootFilesystem: true

tests/integration/overlays/overlay-tests.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Run the integration test that deploys a Sourcegraph cluster with a restrictive security policy.
+# Conveniently, this can be run from dev by setting the TEST_GCP_{ZONE,PROJECT,USERNAME} environment
+# variables (assuming your local `gcloud` is auth'd with the GCP username). Optionally set NOCLEANUP
+# to prevent cleaning up the cluster when finished.
+
+set -xeuo
+
+BUILD_CREATOR="${BUILD_CREATOR:-dev}"
+BUILD_BRANCH="${BUILD_BRANCH:-dev}"
+BUILD_UUID="${BUILD_UUID:-dev}"
+[ ! -z "$TEST_GCP_ZONE" ]
+[ ! -z "$TEST_GCP_PROJECT" ]
+[ ! -z "$GH_TOKEN" ]
+
+CLEANUP=""
+trap 'bash -c "$CLEANUP"' EXIT
+
+CLUSTER_NAME_SUFFIX=$(echo ${BUILD_UUID} | head -c 8)
+CLUSTER_NAME="ds-test-restricted-${CLUSTER_NAME_SUFFIX}"
+# get the STABLE channel version from GKE
+CLUSTER_VERSION=$(gcloud container get-server-config --zone us-central1-a -q 2>&1 | grep "defaultClusterVersion" | awk '{ print $2}')
+
+cd $(dirname "${BASH_SOURCE[0]}")
+
+CURRENT_DIR=$(pwd)
+DEPLOY_SOURCEGRAPH_ROOT=${CURRENT_DIR}/../../..
+
+./install-src.sh
+
+# set up the cluster
+
+gcloud container clusters create ${CLUSTER_NAME} --cluster-version=${CLUSTER_VERSION} --zone ${TEST_GCP_ZONE} --num-nodes 3 --machine-type n1-standard-16 --disk-type pd-ssd --project ${TEST_GCP_PROJECT} --labels="cost-category=build,build-creator=${BUILD_CREATOR},build-branch=${BUILD_BRANCH},integration-test=restricted,repository=deploy-sourcegraph"
+
+gcloud container clusters get-credentials ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT}
+
+# Configure if the test should clean up after itself - useful for debugging
+if [ "${NOCLEANUP:-}" != "true" ]; then
+  CLUSTER_CLEANUP="gcloud container clusters delete ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} --quiet"
+  CLEANUP="$CLUSTER_CLEANUP; $CLEANUP"
+fi
+
+verify() {
+  # hit it with one request
+
+  kubectl -n $NAMESPACE port-forward svc/sourcegraph-frontend 30080 &
+  PROXY=$!
+  CLEANUP="kill $PROXY; $CLEANUP"
+  sleep 2 # (initial delay in port-forward activating)
+  curl --retry-connrefused --retry 2 --retry-delay 10 -m 30 http://localhost:30080
+
+  /usr/local/bin/src version
+
+  # run a validation script against it
+  /usr/local/bin/src -endpoint http://localhost:30080 validate -context github_token=$GH_TOKEN validate.json
+  kill $PROXY
+}
+
+echo "Beginning tests: restricted"
+. restricted.sh
+setup_restricted
+deploy_restricted
+verify
+cleanup_restricted
+
+echo "Beginning tests: base"
+. base.sh
+setup_base
+deploy_base
+verify
+cleanup_base
+
+echo "Beginning tests: storage"
+# FIXME: This is actually failing:
+#statefulsets.apps "gitserver" is forbidden: User "system:serviceaccount:default:sourcegraph-frontend" cannot list resource "statefulsets" in API group "apps" in the namespace "default"
+#. storage.sh
+#setup_storage
+#deploy_storage
+#verify
+#cleanup_storage
+
+echo "Beginning tests: namespaced"
+. namespaced.sh
+setup_namespaced
+deploy_namespaced
+verify
+cleanup_namespaced

tests/integration/overlays/restricted.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+OUTPUT_FOLDER=generated-cluster-restricted
+NAMESPACE=ns-sourcegraph
+
+setup_restricted() {
+  kubectl apply -f sourcegraph.StorageClass.yaml
+
+  kubectl apply -f nonroot-policy.yaml
+
+  kubectl create namespace ${NAMESPACE}
+
+  kubectl create serviceaccount -n ${NAMESPACE} fake-user
+
+  kubectl create rolebinding -n ${NAMESPACE} fake-admin --clusterrole=admin --serviceaccount=${NAMESPACE}:fake-user
+
+  # Kubernetes < 1.16 change to '--resource=podsecuritypolicies.extensions'
+  kubectl create role -n ${NAMESPACE} nonroot:unprivileged --verb=use --resource=podsecuritypolicies.extensions --resource-name=nonroot-policy
+
+  kubectl create rolebinding -n ${NAMESPACE} fake-user:nonroot:unprivileged --role=nonroot:unprivileged --serviceaccount=${NAMESPACE}:fake-user
+
+  /bin/cat <<EOM >deploy_sourcegraph_git_ssh_config
+Host *
+  StrictHostKeyChecking no
+EOM
+
+  kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret
+}
+
+deploy_restricted() {
+  mkdir $OUTPUT_FOLDER
+  CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP"
+  "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh non-privileged-create-cluster ${CURRENT_DIR}/${OUTPUT_FOLDER}
+
+  GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml
+  cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS
+  cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS
+
+  kubectl --as=system:serviceaccount:${NAMESPACE}:fake-user -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080
+
+  # wait for it all to finish (we list out the ones with persistent volume claim because they take longer)
+
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend
+}
+
+cleanup_restricted() {
+  kubectl delete -f sourcegraph.StorageClass.yaml
+
+  kubectl delete -f nonroot-policy.yaml
+
+  kubectl delete namespace ${NAMESPACE}
+}

tests/integration/overlays/sourcegraph.StorageClass.yaml

@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: sourcegraph
+  labels:
+    deploy: sourcegraph-storage
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd # This configures SSDs (recommended).

tests/integration/overlays/storage.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+OUTPUT_FOLDER=generated-cluster-storage
+NAMESPACE=default
+
+setup_storage() {
+  cat sourcegraph.StorageClass.yaml | yj | jq '.metadata.name = "new-storageclass"' | jy -o new-storageclass.StorageClass.yaml
+  kubectl apply -f new-storageclass.StorageClass.yaml
+
+  PVC=${DEPLOY_SOURCEGRAPH_ROOT}/overlays/storageclass/replace-storageclass-name-pvc.yaml
+  cat $PVC | yj | jq '.[].value = "new-storageclass"' | jy -o $PVC
+  PVC=${DEPLOY_SOURCEGRAPH_ROOT}/overlays/storageclass/replace-storageclass-name-sts.yaml
+  cat $PVC | yj | jq '.[].value = "new-storageclass"' | jy -o $PVC
+
+  /bin/cat <<EOM >deploy_sourcegraph_git_ssh_config
+Host *
+  StrictHostKeyChecking no
+EOM
+
+  kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret
+}
+
+deploy_storage() {
+  mkdir $OUTPUT_FOLDER
+  CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP"
+  "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh storageclass ${CURRENT_DIR}/${OUTPUT_FOLDER}
+
+  GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml
+  cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS
+  cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS
+
+  kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080
+
+  # wait for it all to finish (we list out the ones with persistent volume claim because they take longer)
+
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver
+  timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend
+
+  # TODO: Verify that the correct storageclass is being used
+}
+
+cleanup_storage() {
+  kubectl delete -n ${NAMESPACE} -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive
+
+  kubectl delete -f new-storageclass.StorageClass.yaml
+
+  kubectl delete secret -n ${NAMESPACE} gitserver-ssh
+}

tests/integration/overlays/validate.json

@@ -0,0 +1,26 @@
+{
+  "firstAdmin": {
+    "email": "[email protected]",
+    "username": "e2e-test-user",
+    "password": "123123123-e2e-test"
+  },
+  "externalService": {
+    "config": {
+      "url": "https://github.com",
+      "token": "{{ .github_token }}",
+      "orgs": [],
+      "repos": [
+        "sourcegraph-testing/repo-ssh-keys-test"
+      ]
+    },
+    "kind": "GITHUB",
+    "displayName": "footest",
+    "deleteWhenDone": true
+  },
+  "waitRepoCloned": {
+    "repo": "github.com/sourcegraph-testing/repo-ssh-keys-test",
+    "maxTries": 10,
+    "sleepBetweenTriesSeconds": 10
+  },
+  "searchQuery": "repo:^github.com/sourcegraph-testing/repo-ssh-keys-test$ gallery-director-drone-vacant-blizzard"
+}