Kubeconfig provider

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
           chmod +x "${TOOLS_DIR}/bin/bazel"
           curl -Ls -o "${TOOLS_DIR}/bin/buildifier" "https://github.com/bazelbuild/buildtools/releases/download/0.29.0/buildifier"
           chmod +x "${TOOLS_DIR}/bin/buildifier"
-          curl -Ls -o "${TOOLS_DIR}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-linux-amd64"
+          curl -Ls -o "${TOOLS_DIR}/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.10.0/kind-linux-amd64"
           chmod +x "${TOOLS_DIR}/bin/kind"
           echo "${TOOLS_DIR}/bin" >> $GITHUB_PATH
       - name: Setup kind cluster

README.md

@@ -13,11 +13,12 @@ Bazel GitOps Rules is an alternative to [rules_k8s](https://github.com/bazelbuil
 * Speeds up deployments iterations:
   * The results manifests are rendered without pushing containers.
   * Pushes all the images in parallel.
-
+* Provides a utility that creates GitOps pull requests.
 
 ## Rules
 
 * [k8s_deploy](#k8s_deploy)
+* [kubeconfig_configure](#kubeconfig_configure)
 * [k8s_test_setup](#k8s_test_setup)
 
 
@@ -103,7 +104,7 @@ When you run `bazel run ///helloworld:mynamespace.apply`, it applies this file i
 | ***objects***             | `[]`           | A list of other instances of `k8s_deploy` that this one depends on. See [Adding Dependencies](#adding-dependencies).
 | ***images***              | `{}`           | A dict of labels of Docker images. See [Injecting Docker Images](#injecting-docker-images).
 | ***image_digest_tag***    | `False`        | A flag for whether or not to tag the image with the container digest.
-| ***image_registry***      | `docker.io`    | The registry to push images to. 
+| ***image_registry***      | `docker.io`    | The registry to push images to.
 | ***image_repository***    | `None`         | The repository to push images to. By default, this is generated from the current package path.
 | ***image_repository_prefix*** | `None`     | Add a prefix to the image_repository. Can be used to upload the images in
 | ***release_branch_prefix*** | `master`     | A git branch name/prefix. Automatically run GitOps while building this branch. See [GitOps and Deployment](#gitops_and_deployment).
@@ -183,7 +184,8 @@ That looks like a lot. But lets try to decode what is happening here:
 Configmaps are a special case of manifests. They can be rendered from a collection of files of any kind (.yaml, .properties, .xml, .sh, whatever). Let's use hypothetical Grafana deployment as an example:
 
 ```python
-[k8s_deploy(
+[
+    k8s_deploy(
         name = NAME,
         cluster = CLUSTER,
         configmaps_srcs = glob([                 # (1)
@@ -288,7 +290,7 @@ spec:
     - name: java_container
       image: registry.example.com/examples/image@sha256:c94d75d68f4c1b436f545729bbce82774fda07
 ```
-Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example, 
+Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example,
 ```yaml
 apiVersion: v1
 kind: MyCrd
@@ -300,7 +302,7 @@ metadata:
 spec:
   image: "{{//example:my_image}}"
 ```
-would become 
+would become
 ```yaml
 apiVersion: v1
 kind: MyCrd
@@ -343,13 +345,156 @@ Please note that the `objects` attribute is ignored by `.gitops` targets.
 
 
 <a name="gitops_and_deployment"></a>
-### GitOps and Deployment
+## GitOps and Deployment
 
-<!-- TODO(KZ): document gitops and deployment -->
+The GitOps tool is a command line utility that usually runs as a last step of CI pipeline.
 
+For the full list of `create_gitops_prs` command line options, run:
+```bash
+bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs
+```
 
-<a name="k8s_test_setup"></a>
-## k8s_test_setup
+The simplified CI pipeline will look like this:
+```
+[Checkout Code] -> [Bazel Build & Test] -> (if GitOps source branch) -> [Create GitOps PRs]
+```
+
+<a name="trunk_based_gitops_workflow"></a>
+## Trunk Based GitOps Workflow
+
+For example let's assume the CI build pipeline described above is running the build for `https://github.com/example/repo.git`. We are using trunk based branching model. All feature branches are merged into the `master` branch first. The *Create GitOps PRs* step runs on a `master` branch change. The GitOps deployments source files are located in the same repository under the `/cloud` directory.
+
+The *Create GitOps PRs* pipeline step shell command will look like following:
+```bash
+GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
+GIT_COMMIT_ID=$(git rev-parse HEAD)
+GIT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
+if [ "${GIT_BRANCH_NAME}" == "master"]; then
+    bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs -- \
+        --workspace $GIT_ROOT_DIR \
+        --git_repo https://github.com/example/repo.git \
+        --git_mirror $GIT_ROOT_DIR/.git \
+        --git_server github \
+        --release_branch master \
+        --gitops_pr_into master \
+        --branch_name ${GIT_BRANCH_NAME} \
+        --git_commit ${GIT_COMMIT_ID} \
+fi
+```
+
+The `GIT_*` variables describe the current state of the Git repository.
+
+The `--git_repo` parameter defines the remote repository URL. In this case remote repository matches the repository of the working copy. The `--git_mirror` parameter is an optimization used to speed up the target repository clone process using reference repository (see `git clone --reference`). The `--git-server` parameter selects the type of Git server.
+
+The `--release_branch` specifies the value of the ***release_branch_prefix*** attribute of `gitops` targets (see [k8s_deploy](#k8s_deploy)). The `--gitops_pr_into` defines the target branch for newly created pull requests. The `--branch_name` and `--git_commit` are the values used in the pull request commit message.
+
+The `create_gitops_prs` tool will query all `gitops` targets which have set the ***deploy_branch*** attribute (see [k8s_deploy](#k8s_deploy)) and the ***release_branch_prefix*** attribute value that matches the `release_branch` parameter.
+
+The all discovered `gitops` targets are grouped by the value of ***deploy_branch*** attribute. The one deployment branch will accumulate the output of all corresponding `gitops` targets.
+
+For example, we define two deployments: grafana and prometheus. Both deployments share the same namespace. The desired deployment granularity is per namespace.
+```python
+[
+    k8s_deploy(
+        name = NAME,
+        deploy_branch = NAMESPACE,
+        ...
+    )
+    for NAME, CLUSTER, NAMESPACE in [
+        ...
+        ("stage-grafana", "stage", "monitoring-stage"),
+        ("prod-grafana", "prod", "monitoring-prod"),
+    ]
+]
+[
+    k8s_deploy(
+        name = NAME,
+        deploy_branch = NAMESPACE,
+        ...
+    )
+    for NAME, CLUSTER, NAMESPACE in [
+        ...
+        ("stage-prometheus", "stage", "monitoring-stage"),
+        ("prod-prometheus", "prod", "monitoring-prod"),
+    ]
+]
+```
+
+As a result of the setup above the `create_gitops_prs` tool will open up to 2 potential deployment pull requests:
+* from `deploy/monitoring-stage` to `master` including manifests for `stage-grafana` and `stage-prometheus`
+* from `deploy/monitoring-prod` to `master` including manifests for `prod-grafana` and `prod-prometheus`
+
+The GitOps pull request is only created (or new commits added) if the `gitops` target changes the state for the target deployment branch. The source pull request will remain open (and keep accumulation GitOps results) until the pull request is merged and source branch is deleted.
+
+<a name="multiple_release_branches_gitops_workflow"></a>
+## Multiple Release Branches GitOps Workflow
+
+In the situation when the trunk based branching model in not suitable teh `create_gitops_prs` tool supports creating GitOps pull requests before the code is merged to `master` branch.
+
+Both trunk and release branch workflow could coexists in the same repository.
+
+For example, let's assume the CI build pipeline described above is running the build for `https://github.com/example/repo.git`. We are using release branch branching model. Feature request are merged into multiple target release branches. The release brach name convention is `release/team-<YYYYMMDD>`. The *Create GitOps PRs* step is running on the release branch change. GitOps deployments source files are located in the same repository `/cloud` directory in the `master` branch.
+
+The *Create GitOps PRs* pipeline step shell command will look like following:
+```bash
+GIT_ROOT_DIR=$(git rev-parse --show-toplevel)
+GIT_COMMIT_ID=$(git rev-parse HEAD)
+GIT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)          # => release/team-20200101
+RELEASE_BRANCH_SUFFIX=${GIT_BRANCH_NAME#"release/team"}     # => -20200101
+RELEASE_BRANCH=${GIT_BRANCH_NAME%${RELEASE_BRANCH_SUFFIX}}  # => release/team
+if [ "${RELEASE_BRANCH}" == "release/team"]; then
+    bazel run @com_adobe_rules_gitops//gitops/prer:create_gitops_prs -- \
+        --workspace $GIT_ROOT_DIR \
+        --git_repo https://github.com/example/repo.git \
+        --git_mirror $GIT_ROOT_DIR/.git \
+        --git_server github \
+        --release_branch ${RELEASE_BRANCH} \
+        --deployment_branch_suffix=${RELEASE_BRANCH_SUFFIX} \
+        --gitops_pr_into master \
+        --branch_name ${GIT_BRANCH_NAME} \
+        --git_commit ${GIT_COMMIT_ID} \
+fi
+```
+
+The meaning of the parameters is the same as with [trunk based workflow](#trunk_based_gitops_workflow).
+The `--release_branch` parameter takes the value of `release/team`. The additional parameter `--deployment_branch_suffix` will add the release branch suffix to the target deployment branch name.
+
+If we modify previous example:
+```python
+[
+    k8s_deploy(
+        name = NAME,
+        deploy_branch = NAMESPACE,
+        release_branch_prefix = "release/team",  # will be selected only when --release_branch=release/team
+        ...
+    )
+    for NAME, CLUSTER, NAMESPACE in [
+        ...
+        ("stage-grafana", "stage", "monitoring-stage"),
+        ("prod-grafana", "prod", "monitoring-prod"),
+    ]
+]
+[
+    k8s_deploy(
+        name = NAME,
+        deploy_branch = NAMESPACE,
+        release_branch_prefix = "release/team",  # will be selected only when --release_branch=release/team
+        ...
+    )
+    for NAME, CLUSTER, NAMESPACE in [
+        ...
+        ("stage-prometheus", "stage", "monitoring-stage"),
+        ("prod-prometheus", "prod", "monitoring-prod"),
+    ]
+]
+```
+
+The result of the setup above the `create_gitops_prs` tool will open up to 2 potential deployment pull requests per release branch. Assuming release branch name is `release/team-20200101`:
+* from `deploy/monitoring-stage-20200101` to `master` including manifests for `stage-grafana` and `stage-prometheus`
+* from `deploy/monitoring-prod-20200101` to `master` including manifests for `prod-grafana` and `prod-prometheus`
+
+
+## Integration Testing Support
 
 Integration tests are defined in `BUILD` files like this:
 ```python
@@ -381,8 +526,51 @@ The `k8s_test_setup` rule produces a shell script which creates a temporary name
 
 The output of the `k8s_test_setup` rule (a shell script) is referenced in the `java_test` rule. It's listed under the `data` attribute, which declares the target as a dependency, and is included in the jvm flags in this clause: `$(location :service_it.setup)`. The "location" function is specific to Bazel: given a target, it returns the path to the file produced by that target. In this case, it returns the path to the shell script created by our `k8s_test_setup` rule.
 
-The test code launches the script to perform the test setup. The tes code should also monitor the script console output to listen to the pod readiness events.
+The test code launches the script to perform the test setup. The test code should also monitor the script console output to listen to the pod readiness events.
+
+The `@k8s_test//:kubeconfig` target referenced from `k8s_test_setup` rule serves the purpose of making Kubernetes configuration available in the test sandbox. The `kubeconfig_configure` repository rule in the `WORKSPACE` file will need, at minimum, provide the cluster name.
+
+```python
+load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig_configure")
+
+kubeconfig_configure(
+    name = "k8s_test",
+    cluster = "dev",
+)
+```
+
+<a name="k8s_test_setup"></a>
+### k8s_test_setup
+
+**Note:** the `k8s_test_setup` rule is an experimental feature and is subject to change.
+
+An executable that performs Kubernetes test setup:
+
+- creates temporary namespace
+- creates kubectl configuration with the default context set to the created namespace
+- deploys all dependent ***objects***
+- forwards service ports
+
+| Parameter                  | Default        | Description
+| -------------------------- | -------------- | -----------
+| ***kubeconfig***           | `@k8s_test//:kubeconfig` | The Kubernetes configuration file target.
+| ***kubectl***              | `@k8s_test//:kubectl` | The Kubectl executable target.
+| ***objects***              | `None` | A list of other instances of `k8s_deploy` that test depends on. See [Adding Dependencies](#adding-dependencies)
+| ***setup_timeout***        | `10m`  | The time to wait until all required services become ready. The timeout duration should be lower that Bazel test timeout.
+| ***portforward_services*** | `None` | The list of Kubernetes service names to port forward. The setup will wait for at least one service endpoint to become ready.
 
+<a name="kubeconfig_configure"></a>
+### kubeconfig_configure
+
+**Note:** the `kubeconfig_configure` repository rule is an experimental feature and is subject to change.
+
+Configures Kubernetes tools for testing.
+
+| Parameter                 | Default        | Description
+| ------------------------- | -------------- | -----------
+| ***cluster***             | `None`         | The Kubernetes cluster name as defined in the host `kubectl` configuration.
+| ***server***              | `None`         | Optional Kubernetes server endpoint to override automatically detected server endpoint. By default, the server endpoint is automatically detected based on the environment. When running inside the Kubernetes cluster (the service account is present), the server endpoint is derived from `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` environment variables. If environment variable are nto defined the server name is set to `https://kubernetes.default`. Otherwise the host `kubectl` configuration file is used.
+| ***user***                | `None` | Optional Kubernetes configuration user name. Default value is the current build user.
 
 ## Building & Testing
 

create_kind_cluster.sh

@@ -10,6 +10,7 @@
 # governing permissions and limitations under the License.
 
 set -o errexit
+
 # desired cluster name; default is "kind"
 KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
 
@@ -22,16 +23,22 @@ if [ "${running}" != 'true' ]; then
     -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
     registry:2
 fi
-reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
 
-# create a cluster with the local registry enabled in containerd
-cat <<EOF | kind create cluster --name "${KIND_CLUSTER_NAME}" --image "kindest/node:v1.18.15" --config=-
+# create a cluster with the local registry enabled in container
+cat <<EOF | kind create cluster \
+  --name "${KIND_CLUSTER_NAME}" \
+  --image "kindest/node:v1.18.15" \
+  --config=-
+---
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_ip}:${reg_port}"]
+    endpoint = ["http://${reg_name}:${reg_port}"]
 EOF
 
+# connect the registry to the cluster network
+# (the network may already be connected)
+docker network connect "${KIND_CLUSTER_NAME}" "${reg_name}" || true
 

delete_kind_cluster.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2020 Adobe. All rights reserved.
+# This file is licensed to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+# OF ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+set -o errexit
+
+# desired cluster name; default is "kind"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
+
+# delete kind cluster
+kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
+
+# deete registry container
+echo "Deleting kind-registry..."
+docker container rm --force "kind-registry" || true
+
+# delete kind cluster network
+echo "Deleting kind network..."
+docker network rm "kind" || true
+

e2e_test.sh

@@ -31,7 +31,8 @@ if [ "${cluster_running}" != 'true' ]; then
 fi
 
 delete() {
-    echo "Cleanup..."
+  echo "Cleanup..."
+  ./delete_kind_cluster.sh
 }
 
 # Setup a trap to delete the namespace on error

examples/WORKSPACE

@@ -32,3 +32,14 @@ load(
 )
 
 go_image_repositories()
+
+#
+# Kubeconfig repository that will be used in k8s_test_setup
+#
+load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig_configure")
+
+kubeconfig_configure(
+    name = "k8s_test",
+    cluster = "kind-kind",
+    user = "kind-kind",  # kind cluster user name
+)