diff --git a/tests/integration/overlays/base.sh b/tests/integration/overlays/base.sh new file mode 100644 index 000000000000..debcf9de001e --- /dev/null +++ b/tests/integration/overlays/base.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +OUTPUT_FOLDER=generated-cluster-base +NAMESPACE=default + +setup_base() { + kubectl apply -f sourcegraph.StorageClass.yaml + + /bin/cat <deploy_sourcegraph_git_ssh_config +Host * + StrictHostKeyChecking no +EOM + + kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret +} + +deploy_base() { + mkdir $OUTPUT_FOLDER + CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP" + cp -r ${DEPLOY_SOURCEGRAPH_ROOT}/base ${CURRENT_DIR}/${OUTPUT_FOLDER} + + GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/base/gitserver/gitserver.StatefulSet.yaml + cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS + cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS + + kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + # wait for it all to finish (we list out the ones with persistent volume claim because they take longer) + + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend +} + +cleanup_base() { + kubectl delete -n ${NAMESPACE} -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + kubectl delete -f sourcegraph.StorageClass.yaml + + kubectl delete secret -n ${NAMESPACE} gitserver-ssh +} diff --git a/tests/integration/overlays/install-src.sh b/tests/integration/overlays/install-src.sh new file mode 100755 index 000000000000..66c1bdfbd1d9 --- /dev/null +++ b/tests/integration/overlays/install-src.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ ! "$(which src)" ]; then + echo "Installing src to /usr/local/bin" + mkdir -p /tmp/src + cd /tmp/src + wget https://github.com/sourcegraph/src-cli/releases/download/3.21.7/src-cli_3.21.7_linux_amd64.tar.gz + tar xvzf src-cli_3.21.7_linux_amd64.tar.gz + cp src /usr/local/bin/src + chmod a+x /usr/local/bin/src +fi + +if [ ! "$(which jy)" ]; then + echo "Installing jy to /usr/local/bin" + mkdir -p /tmp/jy + cd /tmp/jy + wget https://github.com/sourcegraph/jy/releases/download/v1.0.0/jy-1.0.0-linux-amd64 + cp jy-1.0.0-linux-amd64 /usr/local/bin/jy + chmod a+x /usr/local/bin/jy +fi + +if [ ! "$(which yj)" ]; then + echo "Installing yj to /usr/local/bin" + mkdir -p /tmp/yj + cd /tmp/yj + wget https://github.com/sourcegraph/yj/releases/download/v1.0.0/yj-1.0.0-linux-amd64 + cp yj-1.0.0-linux-amd64 /usr/local/bin/yj + chmod a+x /usr/local/bin/yj +fi diff --git a/tests/integration/overlays/namespaced.sh b/tests/integration/overlays/namespaced.sh new file mode 100644 index 000000000000..6178e4f99076 --- /dev/null +++ b/tests/integration/overlays/namespaced.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +OUTPUT_FOLDER=generated-cluster-namespaced +NAMESPACE=ns-sourcegraph + +setup_namespaced() { + kubectl apply -f sourcegraph.StorageClass.yaml + + kubectl create namespace ${NAMESPACE} + + /bin/cat <deploy_sourcegraph_git_ssh_config +Host * + StrictHostKeyChecking no +EOM + + kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret +} + +deploy_namespaced() { + mkdir $OUTPUT_FOLDER + CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP" + "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh namespaced ${CURRENT_DIR}/${OUTPUT_FOLDER} + + GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml + cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS + cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS + + kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080 + + # wait for it all to finish (we list out the ones with persistent volume claim because they take longer) + + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend +} + +cleanup_namespaced() { + kubectl delete -f sourcegraph.StorageClass.yaml + + kubectl delete namespace ${NAMESPACE} +} diff --git a/tests/integration/overlays/nonroot-policy.yaml b/tests/integration/overlays/nonroot-policy.yaml new file mode 100644 index 000000000000..af9b1629fd73 --- /dev/null +++ b/tests/integration/overlays/nonroot-policy.yaml @@ -0,0 +1,27 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: nonroot-policy +spec: + privileged: false + allowPrivilegeEscalation: false + # The rest fills in some required fields. + seLinux: + rule: RunAsAny + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + runAsUser: + rule: 'MustRunAsNonRoot' + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + volumes: + - '*' + readOnlyRootFilesystem: true diff --git a/tests/integration/overlays/overlay-tests.sh b/tests/integration/overlays/overlay-tests.sh new file mode 100755 index 000000000000..73beb327d181 --- /dev/null +++ b/tests/integration/overlays/overlay-tests.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Run the integration test that deploys a Sourcegraph cluster with a restrictive security policy. +# Conveniently, this can be run from dev by setting the TEST_GCP_{ZONE,PROJECT,USERNAME} environment +# variables (assuming your local `gcloud` is auth'd with the GCP username). Optionally set NOCLEANUP +# to prevent cleaning up the cluster when finished. + +set -xeuo + +BUILD_CREATOR="${BUILD_CREATOR:-dev}" +BUILD_BRANCH="${BUILD_BRANCH:-dev}" +BUILD_UUID="${BUILD_UUID:-dev}" +[ ! -z "$TEST_GCP_ZONE" ] +[ ! -z "$TEST_GCP_PROJECT" ] +[ ! -z "$GH_TOKEN" ] + +CLEANUP="" +trap 'bash -c "$CLEANUP"' EXIT + +CLUSTER_NAME_SUFFIX=$(echo ${BUILD_UUID} | head -c 8) +CLUSTER_NAME="ds-test-restricted-${CLUSTER_NAME_SUFFIX}" +# get the STABLE channel version from GKE +CLUSTER_VERSION=$(gcloud container get-server-config --zone us-central1-a -q 2>&1 | grep "defaultClusterVersion" | awk '{ print $2}') + +cd $(dirname "${BASH_SOURCE[0]}") + +CURRENT_DIR=$(pwd) +DEPLOY_SOURCEGRAPH_ROOT=${CURRENT_DIR}/../../.. + +./install-src.sh + +# set up the cluster + +gcloud container clusters create ${CLUSTER_NAME} --cluster-version=${CLUSTER_VERSION} --zone ${TEST_GCP_ZONE} --num-nodes 3 --machine-type n1-standard-16 --disk-type pd-ssd --project ${TEST_GCP_PROJECT} --labels="cost-category=build,build-creator=${BUILD_CREATOR},build-branch=${BUILD_BRANCH},integration-test=restricted,repository=deploy-sourcegraph" + +gcloud container clusters get-credentials ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} + +# Configure if the test should clean up after itself - useful for debugging +if [ "${NOCLEANUP:-}" != "true" ]; then + CLUSTER_CLEANUP="gcloud container clusters delete ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} --quiet" + CLEANUP="$CLUSTER_CLEANUP; $CLEANUP" +fi + +verify() { + # hit it with one request + + kubectl -n $NAMESPACE port-forward svc/sourcegraph-frontend 30080 & + PROXY=$! + CLEANUP="kill $PROXY; $CLEANUP" + sleep 2 # (initial delay in port-forward activating) + curl --retry-connrefused --retry 2 --retry-delay 10 -m 30 http://localhost:30080 + + /usr/local/bin/src version + + # run a validation script against it + /usr/local/bin/src -endpoint http://localhost:30080 validate -context github_token=$GH_TOKEN validate.json + kill $PROXY +} + +echo "Beginning tests: restricted" +. restricted.sh +setup_restricted +deploy_restricted +verify +cleanup_restricted + +echo "Beginning tests: base" +. base.sh +setup_base +deploy_base +verify +cleanup_base + +echo "Beginning tests: storage" +# FIXME: This is actually failing: +#statefulsets.apps "gitserver" is forbidden: User "system:serviceaccount:default:sourcegraph-frontend" cannot list resource "statefulsets" in API group "apps" in the namespace "default" +#. storage.sh +#setup_storage +#deploy_storage +#verify +#cleanup_storage + +echo "Beginning tests: namespaced" +. namespaced.sh +setup_namespaced +deploy_namespaced +verify +cleanup_namespaced diff --git a/tests/integration/overlays/restricted.sh b/tests/integration/overlays/restricted.sh new file mode 100644 index 000000000000..89effcb14973 --- /dev/null +++ b/tests/integration/overlays/restricted.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +OUTPUT_FOLDER=generated-cluster-restricted +NAMESPACE=ns-sourcegraph + +setup_restricted() { + kubectl apply -f sourcegraph.StorageClass.yaml + + kubectl apply -f nonroot-policy.yaml + + kubectl create namespace ${NAMESPACE} + + kubectl create serviceaccount -n ${NAMESPACE} fake-user + + kubectl create rolebinding -n ${NAMESPACE} fake-admin --clusterrole=admin --serviceaccount=${NAMESPACE}:fake-user + + # Kubernetes < 1.16 change to '--resource=podsecuritypolicies.extensions' + kubectl create role -n ${NAMESPACE} nonroot:unprivileged --verb=use --resource=podsecuritypolicies.extensions --resource-name=nonroot-policy + + kubectl create rolebinding -n ${NAMESPACE} fake-user:nonroot:unprivileged --role=nonroot:unprivileged --serviceaccount=${NAMESPACE}:fake-user + + /bin/cat <deploy_sourcegraph_git_ssh_config +Host * + StrictHostKeyChecking no +EOM + + kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret +} + +deploy_restricted() { + mkdir $OUTPUT_FOLDER + CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP" + "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh non-privileged-create-cluster ${CURRENT_DIR}/${OUTPUT_FOLDER} + + GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml + cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS + cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS + + kubectl --as=system:serviceaccount:${NAMESPACE}:fake-user -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080 + + # wait for it all to finish (we list out the ones with persistent volume claim because they take longer) + + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend +} + +cleanup_restricted() { + kubectl delete -f sourcegraph.StorageClass.yaml + + kubectl delete -f nonroot-policy.yaml + + kubectl delete namespace ${NAMESPACE} +} diff --git a/tests/integration/overlays/sourcegraph.StorageClass.yaml b/tests/integration/overlays/sourcegraph.StorageClass.yaml new file mode 100644 index 000000000000..9a7daaa698cd --- /dev/null +++ b/tests/integration/overlays/sourcegraph.StorageClass.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: sourcegraph + labels: + deploy: sourcegraph-storage +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd # This configures SSDs (recommended). diff --git a/tests/integration/overlays/storage.sh b/tests/integration/overlays/storage.sh new file mode 100644 index 000000000000..55b822ede85e --- /dev/null +++ b/tests/integration/overlays/storage.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +OUTPUT_FOLDER=generated-cluster-storage +NAMESPACE=default + +setup_storage() { + cat sourcegraph.StorageClass.yaml | yj | jq '.metadata.name = "new-storageclass"' | jy -o new-storageclass.StorageClass.yaml + kubectl apply -f new-storageclass.StorageClass.yaml + + PVC=${DEPLOY_SOURCEGRAPH_ROOT}/overlays/storageclass/replace-storageclass-name-pvc.yaml + cat $PVC | yj | jq '.[].value = "new-storageclass"' | jy -o $PVC + PVC=${DEPLOY_SOURCEGRAPH_ROOT}/overlays/storageclass/replace-storageclass-name-sts.yaml + cat $PVC | yj | jq '.[].value = "new-storageclass"' | jy -o $PVC + + /bin/cat <deploy_sourcegraph_git_ssh_config +Host * + StrictHostKeyChecking no +EOM + + kubectl create secret -n ${NAMESPACE} generic gitserver-ssh --from-literal=rsa=supersecret --from-literal=config=topsecret +} + +deploy_storage() { + mkdir $OUTPUT_FOLDER + CLEANUP="rm -rf ${OUTPUT_FOLDER}; $CLEANUP" + "${DEPLOY_SOURCEGRAPH_ROOT}"/overlay-generate-cluster.sh storageclass ${CURRENT_DIR}/${OUTPUT_FOLDER} + + GS=${CURRENT_DIR}/${OUTPUT_FOLDER}/apps_v1_statefulset_gitserver.yaml + cat $GS | yj | jq '.spec.template.spec.containers[].volumeMounts += [{mountPath: "/home/sourcegraph/.ssh", name: "ssh"}]' | jy -o $GS + cat $GS | yj | jq '.spec.template.spec.volumes += [{name: "ssh", secret: {defaultMode: 384, secretName:"gitserver-ssh"}}]' | jy -o $GS + + kubectl -n ${NAMESPACE} apply -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + # kubectl -n ${NAMESPACE} expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer --port=3080 --target-port=3080 + + # wait for it all to finish (we list out the ones with persistent volume claim because they take longer) + + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/indexed-search + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/prometheus + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-cache + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/redis-store + timeout 10m kubectl -n ${NAMESPACE} rollout status -w statefulset/gitserver + timeout 10m kubectl -n ${NAMESPACE} rollout status -w deployment/sourcegraph-frontend + + # TODO: Verify that the correct storageclass is being used +} + +cleanup_storage() { + kubectl delete -n ${NAMESPACE} -f ${CURRENT_DIR}/${OUTPUT_FOLDER} --recursive + + kubectl delete -f new-storageclass.StorageClass.yaml + + kubectl delete secret -n ${NAMESPACE} gitserver-ssh +} diff --git a/tests/integration/overlays/validate.json b/tests/integration/overlays/validate.json new file mode 100644 index 000000000000..8f141c50c5c4 --- /dev/null +++ b/tests/integration/overlays/validate.json @@ -0,0 +1,26 @@ +{ + "firstAdmin": { + "email": "e2e@sourcegrah.com", + "username": "e2e-test-user", + "password": "123123123-e2e-test" + }, + "externalService": { + "config": { + "url": "https://github.com", + "token": "{{ .github_token }}", + "orgs": [], + "repos": [ + "sourcegraph-testing/repo-ssh-keys-test" + ] + }, + "kind": "GITHUB", + "displayName": "footest", + "deleteWhenDone": true + }, + "waitRepoCloned": { + "repo": "github.com/sourcegraph-testing/repo-ssh-keys-test", + "maxTries": 10, + "sleepBetweenTriesSeconds": 10 + }, + "searchQuery": "repo:^github.com/sourcegraph-testing/repo-ssh-keys-test$ gallery-director-drone-vacant-blizzard" +}