diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b31afb16..c80f5263 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,7 +36,7 @@ jobs: chmod +x "${TOOLS_DIR}/bin/bazel" curl -Ls -o "${TOOLS_DIR}/bin/buildifier" "https://github.com/bazelbuild/buildtools/releases/download/0.29.0/buildifier" chmod +x "${TOOLS_DIR}/bin/buildifier" - curl -Ls -o "${TOOLS_DIR}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-linux-amd64" + curl -Ls -o "${TOOLS_DIR}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.10.0/kind-linux-amd64" chmod +x "${TOOLS_DIR}/bin/kind" echo "${TOOLS_DIR}/bin" >> $GITHUB_PATH - name: Setup kind cluster diff --git a/README.md b/README.md index 6d3722f0..c083b98d 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,12 @@ Bazel GitOps Rules is an alternative to [rules_k8s](https://github.com/bazelbuil * Speeds up deployments iterations: * The results manifests are rendered without pushing containers. * Pushes all the images in parallel. - +* Provides a utility that creates GitOps pull requests. ## Rules * [k8s_deploy](#k8s_deploy) +* [kubeconfig_configure](#kubeconfig_configure) * [k8s_test_setup](#k8s_test_setup) @@ -103,7 +104,7 @@ When you run `bazel run ///helloworld:mynamespace.apply`, it applies this file i | ***objects*** | `[]` | A list of other instances of `k8s_deploy` that this one depends on. See [Adding Dependencies](#adding-dependencies). | ***images*** | `{}` | A dict of labels of Docker images. See [Injecting Docker Images](#injecting-docker-images). | ***image_digest_tag*** | `False` | A flag for whether or not to tag the image with the container digest. -| ***image_registry*** | `docker.io` | The registry to push images to. +| ***image_registry*** | `docker.io` | The registry to push images to. | ***image_repository*** | `None` | The repository to push images to. By default, this is generated from the current package path. | ***image_repository_prefix*** | `None` | Add a prefix to the image_repository. Can be used to upload the images in | ***release_branch_prefix*** | `master` | A git branch name/prefix. Automatically run GitOps while building this branch. See [GitOps and Deployment](#gitops_and_deployment). @@ -183,7 +184,8 @@ That looks like a lot. But lets try to decode what is happening here: Configmaps are a special case of manifests. They can be rendered from a collection of files of any kind (.yaml, .properties, .xml, .sh, whatever). Let's use hypothetical Grafana deployment as an example: ```python -[k8s_deploy( +[ + k8s_deploy( name = NAME, cluster = CLUSTER, configmaps_srcs = glob([ # (1) @@ -288,7 +290,7 @@ spec: - name: java_container image: registry.example.com/examples/image@sha256:c94d75d68f4c1b436f545729bbce82774fda07 ``` -Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example, +Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example, ```yaml apiVersion: v1 kind: MyCrd @@ -300,7 +302,7 @@ metadata: spec: image: "{{//example:my_image}}" ``` -would become +would become ```yaml apiVersion: v1 kind: MyCrd @@ -343,13 +345,156 @@ Please note that the `objects` attribute is ignored by `.gitops` targets. -### GitOps and Deployment +## GitOps and Deployment - +The GitOps tool is a command line utility that usually runs as a last step of CI pipeline. +For the full list of `create_gitops_prs` command line options, run: +```bash +bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs +``` - -## k8s_test_setup +The simplified CI pipeline will look like this: +``` +[Checkout Code] -> [Bazel Build & Test] -> (if GitOps source branch) -> [Create GitOps PRs] +``` + + +## Trunk Based GitOps Workflow + +For example let's assume the CI build pipeline described above is running the build for `https://github.com/example/repo.git`. We are using trunk based branching model. All feature branches are merged into the `master` branch first. The *Create GitOps PRs* step runs on a `master` branch change. The GitOps deployments source files are located in the same repository under the `/cloud` directory. + +The *Create GitOps PRs* pipeline step shell command will look like following: +```bash +GIT_ROOT_DIR=$(git rev-parse --show-toplevel) +GIT_COMMIT_ID=$(git rev-parse HEAD) +GIT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) +if [ "${GIT_BRANCH_NAME}" == "master"]; then + bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs -- \ + --workspace $GIT_ROOT_DIR \ + --git_repo https://github.com/example/repo.git \ + --git_mirror $GIT_ROOT_DIR/.git \ + --git_server github \ + --release_branch master \ + --gitops_pr_into master \ + --branch_name ${GIT_BRANCH_NAME} \ + --git_commit ${GIT_COMMIT_ID} \ +fi +``` + +The `GIT_*` variables describe the current state of the Git repository. + +The `--git_repo` parameter defines the remote repository URL. In this case remote repository matches the repository of the working copy. The `--git_mirror` parameter is an optimization used to speed up the target repository clone process using reference repository (see `git clone --reference`). The `--git-server` parameter selects the type of Git server. + +The `--release_branch` specifies the value of the ***release_branch_prefix*** attribute of `gitops` targets (see [k8s_deploy](#k8s_deploy)). The `--gitops_pr_into` defines the target branch for newly created pull requests. The `--branch_name` and `--git_commit` are the values used in the pull request commit message. + +The `create_gitops_prs` tool will query all `gitops` targets which have set the ***deploy_branch*** attribute (see [k8s_deploy](#k8s_deploy)) and the ***release_branch_prefix*** attribute value that matches the `release_branch` parameter. + +The all discovered `gitops` targets are grouped by the value of ***deploy_branch*** attribute. The one deployment branch will accumulate the output of all corresponding `gitops` targets. + +For example, we define two deployments: grafana and prometheus. Both deployments share the same namespace. The desired deployment granularity is per namespace. +```python +[ + k8s_deploy( + name = NAME, + deploy_branch = NAMESPACE, + ... + ) + for NAME, CLUSTER, NAMESPACE in [ + ... + ("stage-grafana", "stage", "monitoring-stage"), + ("prod-grafana", "prod", "monitoring-prod"), + ] +] +[ + k8s_deploy( + name = NAME, + deploy_branch = NAMESPACE, + ... + ) + for NAME, CLUSTER, NAMESPACE in [ + ... + ("stage-prometheus", "stage", "monitoring-stage"), + ("prod-prometheus", "prod", "monitoring-prod"), + ] +] +``` + +As a result of the setup above the `create_gitops_prs` tool will open up to 2 potential deployment pull requests: +* from `deploy/monitoring-stage` to `master` including manifests for `stage-grafana` and `stage-prometheus` +* from `deploy/monitoring-prod` to `master` including manifests for `prod-grafana` and `prod-prometheus` + +The GitOps pull request is only created (or new commits added) if the `gitops` target changes the state for the target deployment branch. The source pull request will remain open (and keep accumulation GitOps results) until the pull request is merged and source branch is deleted. + + +## Multiple Release Branches GitOps Workflow + +In the situation when the trunk based branching model in not suitable teh `create_gitops_prs` tool supports creating GitOps pull requests before the code is merged to `master` branch. + +Both trunk and release branch workflow could coexists in the same repository. + +For example, let's assume the CI build pipeline described above is running the build for `https://github.com/example/repo.git`. We are using release branch branching model. Feature request are merged into multiple target release branches. The release brach name convention is `release/team-`. The *Create GitOps PRs* step is running on the release branch change. GitOps deployments source files are located in the same repository `/cloud` directory in the `master` branch. + +The *Create GitOps PRs* pipeline step shell command will look like following: +```bash +GIT_ROOT_DIR=$(git rev-parse --show-toplevel) +GIT_COMMIT_ID=$(git rev-parse HEAD) +GIT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) # => release/team-20200101 +RELEASE_BRANCH_SUFFIX=${GIT_BRANCH_NAME#"release/team"} # => -20200101 +RELEASE_BRANCH=${GIT_BRANCH_NAME%${RELEASE_BRANCH_SUFFIX}} # => release/team +if [ "${RELEASE_BRANCH}" == "release/team"]; then + bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs -- \ + --workspace $GIT_ROOT_DIR \ + --git_repo https://github.com/example/repo.git \ + --git_mirror $GIT_ROOT_DIR/.git \ + --git_server github \ + --release_branch ${RELEASE_BRANCH} \ + --deployment_branch_suffix=${RELEASE_BRANCH_SUFFIX} \ + --gitops_pr_into master \ + --branch_name ${GIT_BRANCH_NAME} \ + --git_commit ${GIT_COMMIT_ID} \ +fi +``` + +The meaning of the parameters is the same as with [trunk based workflow](#trunk_based_gitops_workflow). +The `--release_branch` parameter takes the value of `release/team`. The additional parameter `--deployment_branch_suffix` will add the release branch suffix to the target deployment branch name. + +If we modify previous example: +```python +[ + k8s_deploy( + name = NAME, + deploy_branch = NAMESPACE, + release_branch_prefix = "release/team", # will be selected only when --release_branch=release/team + ... + ) + for NAME, CLUSTER, NAMESPACE in [ + ... + ("stage-grafana", "stage", "monitoring-stage"), + ("prod-grafana", "prod", "monitoring-prod"), + ] +] +[ + k8s_deploy( + name = NAME, + deploy_branch = NAMESPACE, + release_branch_prefix = "release/team", # will be selected only when --release_branch=release/team + ... + ) + for NAME, CLUSTER, NAMESPACE in [ + ... + ("stage-prometheus", "stage", "monitoring-stage"), + ("prod-prometheus", "prod", "monitoring-prod"), + ] +] +``` + +The result of the setup above the `create_gitops_prs` tool will open up to 2 potential deployment pull requests per release branch. Assuming release branch name is `release/team-20200101`: +* from `deploy/monitoring-stage-20200101` to `master` including manifests for `stage-grafana` and `stage-prometheus` +* from `deploy/monitoring-prod-20200101` to `master` including manifests for `prod-grafana` and `prod-prometheus` + + +## Integration Testing Support Integration tests are defined in `BUILD` files like this: ```python @@ -381,8 +526,51 @@ The `k8s_test_setup` rule produces a shell script which creates a temporary name The output of the `k8s_test_setup` rule (a shell script) is referenced in the `java_test` rule. It's listed under the `data` attribute, which declares the target as a dependency, and is included in the jvm flags in this clause: `$(location :service_it.setup)`. The "location" function is specific to Bazel: given a target, it returns the path to the file produced by that target. In this case, it returns the path to the shell script created by our `k8s_test_setup` rule. -The test code launches the script to perform the test setup. The tes code should also monitor the script console output to listen to the pod readiness events. +The test code launches the script to perform the test setup. The test code should also monitor the script console output to listen to the pod readiness events. + +The `@k8s_test//:kubeconfig` target referenced from `k8s_test_setup` rule serves the purpose of making Kubernetes configuration available in the test sandbox. The `kubeconfig_configure` repository rule in the `WORKSPACE` file will need, at minimum, provide the cluster name. + +```python +load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig_configure") + +kubeconfig_configure( + name = "k8s_test", + cluster = "dev", +) +``` + + +### k8s_test_setup + +**Note:** the `k8s_test_setup` rule is an experimental feature and is subject to change. + +An executable that performs Kubernetes test setup: + +- creates temporary namespace +- creates kubectl configuration with the default context set to the created namespace +- deploys all dependent ***objects*** +- forwards service ports + +| Parameter | Default | Description +| -------------------------- | -------------- | ----------- +| ***kubeconfig*** | `@k8s_test//:kubeconfig` | The Kubernetes configuration file target. +| ***kubectl*** | `@k8s_test//:kubectl` | The Kubectl executable target. +| ***objects*** | `None` | A list of other instances of `k8s_deploy` that test depends on. See [Adding Dependencies](#adding-dependencies) +| ***setup_timeout*** | `10m` | The time to wait until all required services become ready. The timeout duration should be lower that Bazel test timeout. +| ***portforward_services*** | `None` | The list of Kubernetes service names to port forward. The setup will wait for at least one service endpoint to become ready. + +### kubeconfig_configure + +**Note:** the `kubeconfig_configure` repository rule is an experimental feature and is subject to change. + +Configures Kubernetes tools for testing. + +| Parameter | Default | Description +| ------------------------- | -------------- | ----------- +| ***cluster*** | `None` | The Kubernetes cluster name as defined in the host `kubectl` configuration. +| ***server*** | `None` | Optional Kubernetes server endpoint to override automatically detected server endpoint. By default, the server endpoint is automatically detected based on the environment. When running inside the Kubernetes cluster (the service account is present), the server endpoint is derived from `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` environment variables. If environment variable are nto defined the server name is set to `https://kubernetes.default`. Otherwise the host `kubectl` configuration file is used. +| ***user*** | `None` | Optional Kubernetes configuration user name. Default value is the current build user. ## Building & Testing diff --git a/create_kind_cluster.sh b/create_kind_cluster.sh index 5acdc161..7ce361d0 100755 --- a/create_kind_cluster.sh +++ b/create_kind_cluster.sh @@ -10,6 +10,7 @@ # governing permissions and limitations under the License. set -o errexit + # desired cluster name; default is "kind" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" @@ -22,16 +23,22 @@ if [ "${running}" != 'true' ]; then -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ registry:2 fi -reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" -# create a cluster with the local registry enabled in containerd -cat <&2 + +# use BUILD_USER by defalt +USER=${USER:-$BUILD_USER} + +# create miniified self-contained kubectl configuration with the default context set to use newly created namespace +mkdir -p $(dirname $KUBECONFIG_FILE) + +# create context partion of new kubeconfig file from scratch +# use --kubeconfig parameter to prevent any merging +# create +rm -f $KUBECONFIG_FILE-context +CONTEXT=$CLUSTER-$BUILD_USER +kubectl --kubeconfig=$KUBECONFIG_FILE-context --cluster=$CLUSTER --server=$SERVER --user=$USER --namespace=$BUILD_USER config set-context $CONTEXT >&2 +kubectl --kubeconfig=$KUBECONFIG_FILE-context config use-context $CONTEXT >&2 + +# merge newly generated context with system kubeconfig, flatten and minify the result +KUBECONFIG=$KUBECONFIG_FILE-context:$KUBECONFIG kubectl config view --merge=true --minify=true --flatten=true --raw >$KUBECONFIG_FILE + +# set generated kubeconfig for all following kubectl commands +export KUBECONFIG=$KUBECONFIG_FILE + +# check if username from provided configuration exists +KUBECONFIG_USER=$(${KUBECTL} config view -o jsonpath='{.users[?(@.name == '"\"${USER}\")].name}") +if [ -z "${KUBECONFIG_USER}" ]; then + echo "Unable to find user configuration ${USER} for cluster ${CLUSTER}" >&2 + exit 1 +fi + echo "User: ${USER}" >&2 set +e if [ -n "${K8S_MYNAMESPACE:-}" ] then # do not create random namesspace - NAMESPACE=${USER} + NAMESPACE=${BUILD_USER} # do not delete namespace after the test is complete DELETE_NAMESPACE_FLAG="" # do not perform manifest transformations @@ -53,14 +81,19 @@ else DELETE_NAMESPACE_FLAG="-delete_namespace" COUNT="0" while true; do - NAMESPACE=${USER}-$(( (RANDOM) + 32767 )) - ${KUBECTL} --kubeconfig=${KUBECONFIG} --cluster=${CLUSTER} --user=${USER} create namespace ${NAMESPACE} && break + NAMESPACE=${BUILD_USER}-$(( (RANDOM) + 32767 )) + ${KUBECTL} create namespace ${NAMESPACE} && break COUNT=$[$COUNT + 1] if [ $COUNT -ge 10 ]; then echo "Unable to create namespace in $COUNT attempts!" >&2 exit 1 fi done + # update context with created test namespace + kubectl --namespace=$NAMESPACE config set-context $CONTEXT >&2 + + # rename test context (Note: this is required for backward compatibiliy) + kubectl config rename-context $CONTEXT $CLUSTER-$NAMESPACE >&2 fi echo "Namespace: ${NAMESPACE}" >&2 set -e @@ -69,13 +102,7 @@ set -e mkdir -p $(dirname $NAMESPACE_NAME_FILE) echo $NAMESPACE > $NAMESPACE_NAME_FILE -# create kubectl configuration copy with default context set to use newly created namespace -mkdir -p $(dirname $KUBECONFIG_FILE) -cat ${KUBECONFIG} > $KUBECONFIG_FILE -export KUBECONFIG=$KUBECONFIG_FILE -CONTEXT=$CLUSTER-$NAMESPACE -${KUBECTL} --cluster=$CLUSTER --user=$USER --namespace=$NAMESPACE config set-context $CONTEXT >&2 -${KUBECTL} config use-context $CONTEXT >&2 +[ -o xtrace ] && kubectl config view >&2 # set runfiles for STMTS export PYTHON_RUNFILES=${RUNFILES} diff --git a/skylib/kubeconfig.bzl b/skylib/kubeconfig.bzl new file mode 100644 index 00000000..f700d8bf --- /dev/null +++ b/skylib/kubeconfig.bzl @@ -0,0 +1,363 @@ +# Copyright 2021 Adobe. All rights reserved. +# This file is licensed to you under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may obtain a copy +# of the License at http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +# OF ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +""" +Kubectl configuration rules +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//lib:shell.bzl", "shell") + +KubeconfigInfo = provider(fields = [ + "server", + "cluster", + "user", +]) + +def _kubeconfig_impl(ctx): + kubeconfig_file = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = kubeconfig_file, target_file = ctx.file.config) + files = depset(direct = [kubeconfig_file]) + runfiles = ctx.runfiles(files = [kubeconfig_file]) + return [ + DefaultInfo(files = files, runfiles = runfiles), + KubeconfigInfo( + server = ctx.attr.server, + cluster = ctx.attr.cluster, + user = ctx.attr.user, + ), + ] + +kubeconfig = rule( + implementation = _kubeconfig_impl, + attrs = { + "config": attr.label( + doc = "Config file.", + allow_single_file = True, + mandatory = True, + ), + "server": attr.string( + doc = "Optional Kubernetes server url.", + mandatory = False, + ), + "cluster": attr.string( + doc = "Optional Kubernetes cluster name.", + mandatory = False, + ), + "user": attr.string( + doc = "Optional Kubernetes user name.", + mandatory = True, + ), + }, + provides = [DefaultInfo, KubeconfigInfo], +) + +_KUBECONFIG_CONTEXT_TEMPLATE = """ +apiVersion: v1 +kind: Config +contexts: +- context: + cluster: {cluster} + user: {user} + name: {cluster}-{user} +current-context: {cluster}-{user} +""" + +_KUBECONFIG_BUILD_TEMPLATE = """# Generated by kubeconfig_configure repostiory rule + +load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig") + +exports_files(["kubectl"]) + +kubeconfig( + name = "kubeconfig", + config = ":config", + server = "{server}", + cluster = "{cluster}", + user = "{user}", + visibility = ["//visibility:public"], +) +""" + +# kubectl template +def _kubectl_config(repository_ctx, kubeconfig, args): + kubectl = repository_ctx.path("kubectl") + exec_result = repository_ctx.execute( + [kubectl, "--kubeconfig", kubeconfig, "config"] + args, + environment = { + # prevent kubectl config to stumble on shared .kube/config.lock file + "HOME": str(repository_ctx.path(".")), + }, + quiet = True, + ) + if exec_result.return_code != 0: + fail("Error executing kubectl config %s" % " ".join(args)) + return exec_result.stdout + +def _kubectl_config_query(repository_ctx, kubeconfig, jsonpath): + kubectl = repository_ctx.path("kubectl") + query = ["view", "--raw", "-o", "jsonpath=%s" % shell.quote(jsonpath)] + exec_result = repository_ctx.execute( + [kubectl, "--kubeconfig", kubeconfig, "config"] + query, + environment = { + # prevent kubectl config to stumble on shared .kube/config.lock file + "HOME": str(repository_ctx.path(".")), + }, + quiet = True, + ) + if exec_result.return_code != 0: + fail("Error executing kubectl config %s" % " ".join(query)) + return exec_result.stdout.strip("\"\'") + +def _real_kubeconfig_path(repository_ctx, kubeconfig, path): + return repository_ctx.path(paths.normalize(paths.join(str(kubeconfig.dirname), str(path)))) + +def _kubeconfig_configure_impl(repository_ctx): + """Find local kubernetes certificates""" + + # find and symlink kubectl + kubectl = repository_ctx.which("kubectl") + if not kubectl: + fail("Unable to find kubectl executable. PATH=%s" % repository_ctx.path) + repository_ctx.symlink(kubectl, "kubectl") + + home = repository_ctx.path(repository_ctx.os.environ["HOME"]) + + # use provided user name or fall back to current os user name + if repository_ctx.attr.user: + user = repository_ctx.attr.user + elif "USER" in repository_ctx.os.environ: + user = repository_ctx.os.environ["USER"] + else: + exec_result = repository_ctx.execute(["whoami"]) + if exec_result.return_code != 0: + fail("Error detecting current user") + user = exec_result.stdout.rstrip() + + can_symlink = True + token = None + ca_crt = None + kubecert_cert = None + kubecert_key = None + server = repository_ctx.attr.server + host_kubeconfig = None + + # check service account first + serviceaccount = repository_ctx.path("/var/run/secrets/kubernetes.io/serviceaccount") + if serviceaccount.exists: + ca_crt = repository_ctx.path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + token_file = serviceaccount.get_child("token") + if token_file.exists: + exec_result = repository_ctx.execute(["cat", token_file.realpath]) + if exec_result.return_code != 0: + fail("Error reading user token") + token = exec_result.stdout.rstrip() + + # use master url from the environemnt + if "KUBERNETES_SERVICE_HOST" in repository_ctx.os.environ: + server = "https://%s:%s" % ( + repository_ctx.os.environ["KUBERNETES_SERVICE_HOST"], + repository_ctx.os.environ["KUBERNETES_SERVICE_PORT"], + ) + else: + # fall back to the default + server = "https://kubernetes.default" + + # we can't possibly have a kubeconfig file to symlink + can_symlink = False + else: + # check host kubectl configuration file + host_kubeconfig = home.get_child(".kube").get_child("config") + if host_kubeconfig.exists: + # resolve posible relative path values + # the test setup script will symlink kubeconfig file into the test runfiles subtree. + # the relative path resulution will break. + ca_crt = _kubectl_config_query( + repository_ctx, + host_kubeconfig.realpath, + "{.clusters[?(@.name == \"%s\")].cluster.certificate-authority}" % repository_ctx.attr.cluster, + ) + if ca_crt: + if not paths.is_absolute(str(ca_crt)): + ca_crt = _real_kubeconfig_path(repository_ctx, host_kubeconfig, ca_crt) + can_symlink = False + if not server: + # read the cluster server url if not specified + # to be able to update kubeconfig later we will need bouth ca_crt and server values + server = _kubectl_config_query( + repository_ctx, + host_kubeconfig.realpath, + "{.clusters[?(@.name == \"%s\")].cluster.server}" % repository_ctx.attr.cluster, + ) + else: + ca_crt = repository_ctx.path(ca_crt) # requred to be a path object later + + kubecert_cert = _kubectl_config_query( + repository_ctx, + host_kubeconfig.realpath, + "{.users[?(@.name == \"%s\")].user.client-certificate}" % user, + ) + if kubecert_cert: + if not paths.is_absolute(str(kubecert_cert)): + kubecert_cert = _real_kubeconfig_path(repository_ctx, host_kubeconfig, kubecert_cert) + can_symlink = False + else: + kubecert_cert = repository_ctx.path(kubecert_cert) # requred to be a path object later + + kubecert_key = _kubectl_config_query( + repository_ctx, + host_kubeconfig.realpath, + "{.users[?(@.name == \"%s\")].user.client-key}" % user, + ) + if kubecert_key: + if not paths.is_absolute(str(kubecert_key)): + kubecert_key = _real_kubeconfig_path(repository_ctx, host_kubeconfig, kubecert_key) + can_symlink = False + else: + kubecert_key = repository_ctx.path(kubecert_key) # requred to be a path object later + + else: + # Fallback to the legacy behavior used by AdCloud + # Client certs were located at well known paths + certs = home.get_child(".kube").get_child("certs") + ca_crt = certs.get_child("ca.crt") + kubecert_cert = certs.get_child("kubecert.cert") + kubecert_key = certs.get_child("kubecert.key") + + # kubeconfig file doesn't exists + can_symlink = False + + if can_symlink and host_kubeconfig and host_kubeconfig.exists: + # symlink ~/.kube/config if this file exists + repository_ctx.symlink(host_kubeconfig, repository_ctx.path("config")) + else: + # crete new config file service account token or certificates + kubeconfig = repository_ctx.path("config") + + # system kubeconfig file can still exists here + # if that is the case, we try to copy user and cluster settings from the system configuration + if host_kubeconfig and host_kubeconfig.exists: + # create a context file that will allow us use kubectl --mininfy=true + kubeconfig_context = repository_ctx.path("kubeconfig-context") + repository_ctx.file( + kubeconfig_context, + _KUBECONFIG_CONTEXT_TEMPLATE.format( + cluster = repository_ctx.attr.cluster, + user = user, + ), + executable = False, + ) + + # merge system kubeconfig with the newly created context configuration + exec_result = repository_ctx.execute( + [kubectl, "config", "view", "--merge=true", "--minify=true", "--raw"], + environment = { + # enable merge + "KUBECONFIG": "%s:%s" % (kubeconfig_context, host_kubeconfig.realpath), + # prevent kubectl config to stumble on shared .kube/config.lock file + "HOME": str(repository_ctx.path(".")), + }, + quiet = True, + ) + if exec_result.return_code != 0: + fail("Error executing kubectl config view --merge=true --minify=true --raw") + + # write merged kubeconfig + repository_ctx.file( + kubeconfig, + exec_result.stdout, + executable = False, + ) + + # remove context file, it must not be left behind + repository_ctx.delete(kubeconfig_context) + + # Update server endpoint. Ether service account is found or server is passed as a parameter. + # config set-cluster {cluster} \ + # --certificate-authority=... \ + # --server=https://dev3.k8s.tubemogul.info:443 \ + # ", + if server and ca_crt and ca_crt.exists: + _kubectl_config(repository_ctx, kubeconfig, [ + "set-cluster", + repository_ctx.attr.cluster, + "--server", + server, + "--certificate-authority", + ca_crt.realpath, # must be an absolute path + ]) + + # Update user credentials token in case the service account is found. + # config set-credentials {user} --token=...", + if token: + _kubectl_config(repository_ctx, kubeconfig, [ + "set-credentials", + user, + "--token", + token, + ]) + + # Update user credentials certificate. + # config set-credentials {user} --client-certificate=...", + if kubecert_cert and kubecert_cert.exists: + _kubectl_config(repository_ctx, kubeconfig, [ + "set-credentials", + user, + "--client-certificate", + kubecert_cert.realpath, # must be an absolute path + ]) + + # Update user credentials certificate. + # config set-credentials {user} --client-key=...", + if kubecert_key and kubecert_key.exists: + _kubectl_config(repository_ctx, kubeconfig, [ + "set-credentials", + user, + "--client-key", + kubecert_key.realpath, # must be an absolute path + ]) + + # export repostory contents + repository_ctx.file("BUILD", _KUBECONFIG_BUILD_TEMPLATE.format( + cluster = repository_ctx.attr.cluster, + server = repository_ctx.attr.server, + user = user, + ), False) + + return { + "cluster": repository_ctx.attr.cluster, + "server": repository_ctx.attr.server, + "user": user, + } + +kubeconfig_configure = repository_rule( + attrs = { + "cluster": attr.string( + doc = "The Kubernetes cluster name as defined in the host kubectl configuration.", + mandatory = True, + ), + "server": attr.string( + doc = "Kubernetes server endpoint.", + mandatory = False, + ), + "user": attr.string( + doc = "Kubernetes configuration user name.", + mandatory = False, + ), + }, + environ = [ + "HOME", + "USER", + "KUBERNETES_SERVICE_HOST", + "KUBERNETES_SERVICE_PORT", + ], + local = True, + implementation = _kubeconfig_configure_impl, +) diff --git a/skylib/kustomize/BUILD b/skylib/kustomize/BUILD index 01a85ef7..6798da23 100644 --- a/skylib/kustomize/BUILD +++ b/skylib/kustomize/BUILD @@ -17,6 +17,9 @@ exports_files([ sh_binary( name = "set_namespace", srcs = ["set_namespace.sh"], + args = [ + "$(location @kustomize_bin//:kustomize)", + ], data = [ "@kustomize_bin//:kustomize", ], diff --git a/skylib/kustomize/set_namespace.sh b/skylib/kustomize/set_namespace.sh index 203159be..b18bf846 100755 --- a/skylib/kustomize/set_namespace.sh +++ b/skylib/kustomize/set_namespace.sh @@ -11,17 +11,19 @@ set +x -if [ "$1" == "" ]; then +if [ "$2" == "" ]; then echo usage: - echo $0 'namespace out.yaml' + echo $0 'kustomize_bin namespace out.yaml' exit 1 fi set -euo pipefail +kustomize_bin=$1 +namespace=$2 dir=$(mktemp -d) cat >${dir}/in.yaml cat >${dir}/kustomization.yaml <