[YUNIKORN-2135] Add integration test code coverage

Signed-off-by: PoAn Yang <[email protected]>

.github/workflows/pre-commit.yml

@@ -32,6 +32,7 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           files: build/coverage.txt
+          flags: unittests
 
   e2e-tests:
     needs: build
@@ -59,3 +60,46 @@ jobs:
         env:
           KIND_NODE_IMAGE: ${{ matrix.k8s }}
           KIND_EXTRA_ARGS: ${{ matrix.plugin }}
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-cover-dir-merged
+          path: go-cover-dir-merged/
+          if-no-files-found: 'ignore'
+
+  e2e-tests-code-coverage:
+    needs: e2e-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: .go_version
+      - name: Download code-coverage-report artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: downloaded_artifacts
+      - run: |
+          if [ -d downloaded_artifacts/go-cover-dir-merged ];
+          then
+            if [ "$(ls -A downloaded_artifacts/go-cover-dir-merged)" ];
+            then
+              go tool covdata textfmt -i=downloaded_artifacts/go-cover-dir-merged -o yunikorn-k8shim.txt -pkg=github.com/apache/yunikorn-k8shim/...
+              go tool covdata textfmt -i=downloaded_artifacts/go-cover-dir-merged -o yunikorn-core.txt -pkg=github.com/apache/yunikorn-core/...
+            else
+              echo "No files found in downloaded_artifacts/go-cover-dir-merged folder. Skip the rest of the steps."
+              exit 0
+            fi
+          else
+            echo "No downloaded_artifacts/go-cover-dir-merged folder found. Skip the rest of the steps."
+            exit 0
+          fi
+      - name: Code coverage
+        uses: codecov/codecov-action@v3
+        with:
+          files: yunikorn-k8shim.txt
+          flags: e2e

.github/workflows/push-master.yml

@@ -23,3 +23,4 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           files: build/coverage.txt
+          flags: unittests

Makefile

@@ -44,6 +44,14 @@ endif
 # Make sure we are in the same directory as the Makefile
 BASE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
+# Set cover flags for go build and uid/gid for dockerfile
+ifeq ($(ENABLE_GO_COVER_DIR), TRUE)
+	DOCKERFILE_UID_ARG := --build-arg UID=0
+	DOCKERFILE_GID_ARG := --build-arg GID=0
+	COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/...,github.com/apache/yunikorn-core/...
+	ADMISSION_COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/...
+endif
+
 # Output directories
 OUTPUT=build
 DEV_BIN_DIR=${OUTPUT}/dev
@@ -340,7 +348,7 @@ $(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY): go.mod go.sum $(shell find pkg)
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=false -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/shim/
+	${COVER_FLAGS} ./pkg/cmd/shim/
 
 # Build plugin binary in a production ready version
 .PHONY: plugin
@@ -356,7 +364,7 @@ $(RELEASE_BIN_DIR)/$(PLUGIN_BINARY): go.mod go.sum $(shell find pkg)
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=true -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/schedulerplugin/
+	${COVER_FLAGS} ./pkg/cmd/schedulerplugin/
 
 # Build a scheduler image based on the production ready version
 .PHONY: sched_image
@@ -366,7 +374,7 @@ sched_image: scheduler docker/scheduler
 	@mkdir -p "$(DOCKER_DIR)/scheduler"
 	@cp -a "docker/scheduler/." "$(DOCKER_DIR)/scheduler/."
 	@cp "$(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY)" "$(DOCKER_DIR)/scheduler/."
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/scheduler" \
 	-t "$(SCHEDULER_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \
@@ -386,7 +394,7 @@ plugin_image: plugin docker/plugin conf/scheduler-config.yaml
 	@cp -a "docker/plugin/." "$(DOCKER_DIR)/plugin/."
 	@cp "$(RELEASE_BIN_DIR)/$(PLUGIN_BINARY)" "$(DOCKER_DIR)/plugin/."
 	@cp conf/scheduler-config.yaml "$(DOCKER_DIR)/plugin/scheduler-config.yaml"
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/plugin" \
 	-t "$(PLUGIN_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \
@@ -411,7 +419,7 @@ $(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY): go.mod go.sum $(shell find pk
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/admissioncontroller
+	${ADMISSION_COVER_FLAGS} ./pkg/cmd/admissioncontroller
 
 # Build an admission controller image based on the production ready version
 .PHONY: adm_image
@@ -421,7 +429,7 @@ adm_image: admission docker/admission
 	@mkdir -p "$(DOCKER_DIR)/admission"
 	@cp -a "docker/admission/." "$(DOCKER_DIR)/admission/."
 	@cp "$(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY)" "$(DOCKER_DIR)/admission/."
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/admission" \
 	-t "$(ADMISSION_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \

docker/admission/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-admission-controller /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-admission-controller" ]

docker/plugin/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-scheduler-plugin scheduler-config.yaml /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-scheduler-plugin", "--bind-address=0.0.0.0", "--config=/scheduler-config.yaml" ]

docker/scheduler/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-scheduler /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-scheduler" ]

scripts/run-e2e-tests.sh

@@ -87,7 +87,8 @@ function install_cluster() {
   if [ "${GIT_CLONE}" = "true" ]; then
     check_cmd "git"
     rm -rf ./build/yunikorn-release
-    git clone --depth 1 https://github.com/apache/yunikorn-release.git ./build/yunikorn-release
+    # TODO: update following branch if PR in yunikorn-release is merged
+    git clone --depth 1 https://github.com/FrankYang0529/yunikorn-release.git -b YUNIKORN-2135 ./build/yunikorn-release
   fi
   if [ ! -d "${CHART_PATH}" ]; then
     exit_on_error "helm charts not found in path: ${CHART_PATH}"
@@ -96,7 +97,7 @@ function install_cluster() {
   # build docker images from latest code, so that we can install yunikorn with these latest images
   echo "step 3/7: building docker images from latest code"
   check_docker
-  QUIET="--quiet" REGISTRY=local VERSION=latest make image
+  QUIET="--quiet" REGISTRY=local VERSION=latest ENABLE_GO_COVER_DIR=TRUE make image
   exit_on_error "build docker images failed"
   QUIET="--quiet" REGISTRY=local VERSION=latest make webtest_image
   exit_on_error "build test web images failed"
@@ -136,14 +137,45 @@ function install_cluster() {
     --set admissionController.image.pullPolicy=IfNotPresent \
     --set web.image.repository=local/yunikorn \
     --set web.image.tag="${WEBTEST_IMAGE}" \
-    --set web.image.pullPolicy=IfNotPresent
+    --set web.image.pullPolicy=IfNotPresent \
+    --set enableGoCoverDir=true
   exit_on_error "failed to install yunikorn"
   "${KUBECTL}" wait --for=condition=available --timeout=300s deployment/yunikorn-scheduler -n yunikorn
   exit_on_error "failed to wait for yunikorn scheduler deployment being deployed"
   "${KUBECTL}" wait --for=condition=ready --timeout=300s pod -l app=yunikorn -n yunikorn
   exit_on_error "failed to wait for yunikorn scheduler pods being deployed"
 }
 
+function uninstall_yunikorn() {
+  echo "uninstall yunikorn"
+  install_tools
+  "${HELM}" uninstall yunikorn --namespace yunikorn --wait --cascade foreground
+  exit_on_error "failed to uninstall yunikorn"
+}
+
+function copy_go_cover_dir() {
+  echo "copy coverage profile files from kind containers to local go-cover-dir-merged directory"
+  install_tools
+  mkdir -p go-cover-dir-merged
+  GO_COVER_DIR_FOLDERS=()
+  for i in $("${KIND}" get nodes --name "${CLUSTER_NAME}"); do
+    if docker exec -i "$i" test -d /go-cover-dir
+    then
+      docker cp "$i":/go-cover-dir "$i"
+      GO_COVER_DIR_FOLDERS+=("$i")
+    fi
+  done
+
+  if [ ${#GO_COVER_DIR_FOLDERS[@]} -ne 0 ]; then
+    GO_COVER_DIR_STR=$(printf ",%s" "${GO_COVER_DIR_FOLDERS[@]}")
+    GO_COVER_DIR_STR=${GO_COVER_DIR_STR:1}
+    go tool covdata merge -i="${GO_COVER_DIR_STR}" -o go-cover-dir-merged
+    for i in "${GO_COVER_DIR_FOLDERS[@]}"; do
+      rm -rf "$i"
+    done
+  fi
+}
+
 function delete_cluster() {
   echo "deleting K8s cluster: ${CLUSTER_NAME}"
   install_tools
@@ -269,6 +301,8 @@ if [ "${ACTION}" == "test" ]; then
   fi
   make e2e_test
   exit_on_error "e2e tests failed"
+  uninstall_yunikorn
+  copy_go_cover_dir
 elif [ "${ACTION}" == "install" ]; then
   check_cmd "${GO}"
   check_opt "kind-node-image-version" "${CLUSTER_VERSION}"