From 3f1e58d2976787b605fe6188086217fe1547bb0f Mon Sep 17 00:00:00 2001 From: PoAn Yang Date: Sun, 26 Nov 2023 14:30:12 +0800 Subject: [PATCH] [YUNIKORN-2135] Add integration test code coverage Signed-off-by: PoAn Yang --- .github/workflows/pre-commit.yml | 44 +++++++++++++++++++++++++++++++ .github/workflows/push-master.yml | 1 + Makefile | 20 +++++++++----- docker/admission/Dockerfile | 6 ++++- docker/plugin/Dockerfile | 6 ++++- docker/scheduler/Dockerfile | 6 ++++- scripts/run-e2e-tests.sh | 40 +++++++++++++++++++++++++--- 7 files changed, 111 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 67aead9d4..4dd52ac46 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -32,6 +32,7 @@ jobs: uses: codecov/codecov-action@v3 with: files: build/coverage.txt + flags: unittests e2e-tests: needs: build @@ -59,3 +60,46 @@ jobs: env: KIND_NODE_IMAGE: ${{ matrix.k8s }} KIND_EXTRA_ARGS: ${{ matrix.plugin }} + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + with: + name: go-cover-dir-merged + path: go-cover-dir-merged/ + if-no-files-found: 'ignore' + + e2e-tests-code-coverage: + needs: e2e-tests + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v3 + with: + fetch-depth: 2 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: .go_version + - name: Download code-coverage-report artifacts + uses: actions/download-artifact@v3 + with: + path: downloaded_artifacts + - run: | + if [ -d downloaded_artifacts/go-cover-dir-merged ]; + then + if [ "$(ls -A downloaded_artifacts/go-cover-dir-merged)" ]; + then + go tool covdata textfmt -i=downloaded_artifacts/go-cover-dir-merged -o yunikorn-k8shim.txt -pkg=github.com/apache/yunikorn-k8shim/... + go tool covdata textfmt -i=downloaded_artifacts/go-cover-dir-merged -o yunikorn-core.txt -pkg=github.com/apache/yunikorn-core/... + else + echo "No files found in downloaded_artifacts/go-cover-dir-merged folder. Skip the rest of the steps." + exit 0 + fi + else + echo "No downloaded_artifacts/go-cover-dir-merged folder found. Skip the rest of the steps." + exit 0 + fi + - name: Code coverage + uses: codecov/codecov-action@v3 + with: + files: yunikorn-k8shim.txt + flags: e2e diff --git a/.github/workflows/push-master.yml b/.github/workflows/push-master.yml index d79bb2475..28dec370f 100644 --- a/.github/workflows/push-master.yml +++ b/.github/workflows/push-master.yml @@ -23,3 +23,4 @@ jobs: uses: codecov/codecov-action@v3 with: files: build/coverage.txt + flags: unittests diff --git a/Makefile b/Makefile index 4a642a525..b8c4f7f3c 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,14 @@ endif # Make sure we are in the same directory as the Makefile BASE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +# Set cover flags for go build and uid/gid for dockerfile +ifeq ($(ENABLE_GO_COVER_DIR), TRUE) + DOCKERFILE_UID_ARG := --build-arg UID=0 + DOCKERFILE_GID_ARG := --build-arg GID=0 + COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/...,github.com/apache/yunikorn-core/... + ADMISSION_COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/... +endif + # Output directories OUTPUT=build DEV_BIN_DIR=${OUTPUT}/dev @@ -340,7 +348,7 @@ $(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY): go.mod go.sum $(shell find pkg) -ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=false -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \ -tags netgo \ -installsuffix netgo \ - ./pkg/cmd/shim/ + ${COVER_FLAGS} ./pkg/cmd/shim/ # Build plugin binary in a production ready version .PHONY: plugin @@ -356,7 +364,7 @@ $(RELEASE_BIN_DIR)/$(PLUGIN_BINARY): go.mod go.sum $(shell find pkg) -ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=true -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \ -tags netgo \ -installsuffix netgo \ - ./pkg/cmd/schedulerplugin/ + ${COVER_FLAGS} ./pkg/cmd/schedulerplugin/ # Build a scheduler image based on the production ready version .PHONY: sched_image @@ -366,7 +374,7 @@ sched_image: scheduler docker/scheduler @mkdir -p "$(DOCKER_DIR)/scheduler" @cp -a "docker/scheduler/." "$(DOCKER_DIR)/scheduler/." @cp "$(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY)" "$(DOCKER_DIR)/scheduler/." - DOCKER_BUILDKIT=1 docker build \ + DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \ "$(DOCKER_DIR)/scheduler" \ -t "$(SCHEDULER_TAG)" \ --platform "linux/${DOCKER_ARCH}" \ @@ -386,7 +394,7 @@ plugin_image: plugin docker/plugin conf/scheduler-config.yaml @cp -a "docker/plugin/." "$(DOCKER_DIR)/plugin/." @cp "$(RELEASE_BIN_DIR)/$(PLUGIN_BINARY)" "$(DOCKER_DIR)/plugin/." @cp conf/scheduler-config.yaml "$(DOCKER_DIR)/plugin/scheduler-config.yaml" - DOCKER_BUILDKIT=1 docker build \ + DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \ "$(DOCKER_DIR)/plugin" \ -t "$(PLUGIN_TAG)" \ --platform "linux/${DOCKER_ARCH}" \ @@ -411,7 +419,7 @@ $(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY): go.mod go.sum $(shell find pk -ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH}' \ -tags netgo \ -installsuffix netgo \ - ./pkg/cmd/admissioncontroller + ${ADMISSION_COVER_FLAGS} ./pkg/cmd/admissioncontroller # Build an admission controller image based on the production ready version .PHONY: adm_image @@ -421,7 +429,7 @@ adm_image: admission docker/admission @mkdir -p "$(DOCKER_DIR)/admission" @cp -a "docker/admission/." "$(DOCKER_DIR)/admission/." @cp "$(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY)" "$(DOCKER_DIR)/admission/." - DOCKER_BUILDKIT=1 docker build \ + DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \ "$(DOCKER_DIR)/admission" \ -t "$(ADMISSION_TAG)" \ --platform "linux/${DOCKER_ARCH}" \ diff --git a/docker/admission/Dockerfile b/docker/admission/Dockerfile index 2a5fa879d..f45670633 100644 --- a/docker/admission/Dockerfile +++ b/docker/admission/Dockerfile @@ -14,7 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +ARG UID=4444 +ARG GID=4444 + FROM --platform=$TARGETPLATFORM scratch COPY --chown=0:0 yunikorn-admission-controller / -USER 4444:4444 +USER ${UID}:${GID} ENTRYPOINT [ "/yunikorn-admission-controller" ] diff --git a/docker/plugin/Dockerfile b/docker/plugin/Dockerfile index 62580210c..acf08afa6 100644 --- a/docker/plugin/Dockerfile +++ b/docker/plugin/Dockerfile @@ -14,7 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +ARG UID=4444 +ARG GID=4444 + FROM --platform=$TARGETPLATFORM scratch COPY --chown=0:0 yunikorn-scheduler-plugin scheduler-config.yaml / -USER 4444:4444 +USER ${UID}:${GID} ENTRYPOINT [ "/yunikorn-scheduler-plugin", "--bind-address=0.0.0.0", "--config=/scheduler-config.yaml" ] diff --git a/docker/scheduler/Dockerfile b/docker/scheduler/Dockerfile index dacca8017..0956139e7 100644 --- a/docker/scheduler/Dockerfile +++ b/docker/scheduler/Dockerfile @@ -14,7 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +ARG UID=4444 +ARG GID=4444 + FROM --platform=$TARGETPLATFORM scratch COPY --chown=0:0 yunikorn-scheduler / -USER 4444:4444 +USER ${UID}:${GID} ENTRYPOINT [ "/yunikorn-scheduler" ] diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index f245ab1a0..f4d5a0814 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -87,7 +87,8 @@ function install_cluster() { if [ "${GIT_CLONE}" = "true" ]; then check_cmd "git" rm -rf ./build/yunikorn-release - git clone --depth 1 https://github.com/apache/yunikorn-release.git ./build/yunikorn-release + # TODO: update following branch if PR in yunikorn-release is merged + git clone --depth 1 https://github.com/FrankYang0529/yunikorn-release.git -b YUNIKORN-2135 ./build/yunikorn-release fi if [ ! -d "${CHART_PATH}" ]; then exit_on_error "helm charts not found in path: ${CHART_PATH}" @@ -96,7 +97,7 @@ function install_cluster() { # build docker images from latest code, so that we can install yunikorn with these latest images echo "step 3/7: building docker images from latest code" check_docker - QUIET="--quiet" REGISTRY=local VERSION=latest make image + QUIET="--quiet" REGISTRY=local VERSION=latest ENABLE_GO_COVER_DIR=TRUE make image exit_on_error "build docker images failed" QUIET="--quiet" REGISTRY=local VERSION=latest make webtest_image exit_on_error "build test web images failed" @@ -136,7 +137,8 @@ function install_cluster() { --set admissionController.image.pullPolicy=IfNotPresent \ --set web.image.repository=local/yunikorn \ --set web.image.tag="${WEBTEST_IMAGE}" \ - --set web.image.pullPolicy=IfNotPresent + --set web.image.pullPolicy=IfNotPresent \ + --set enableGoCoverDir=true exit_on_error "failed to install yunikorn" "${KUBECTL}" wait --for=condition=available --timeout=300s deployment/yunikorn-scheduler -n yunikorn exit_on_error "failed to wait for yunikorn scheduler deployment being deployed" @@ -144,6 +146,36 @@ function install_cluster() { exit_on_error "failed to wait for yunikorn scheduler pods being deployed" } +function uninstall_yunikorn() { + echo "uninstall yunikorn" + install_tools + "${HELM}" uninstall yunikorn --namespace yunikorn --wait --cascade foreground + exit_on_error "failed to uninstall yunikorn" +} + +function copy_go_cover_dir() { + echo "copy coverage profile files from kind containers to local go-cover-dir-merged directory" + install_tools + mkdir -p go-cover-dir-merged + GO_COVER_DIR_FOLDERS=() + for i in $("${KIND}" get nodes --name "${CLUSTER_NAME}"); do + if docker exec -i "$i" test -d /go-cover-dir + then + docker cp "$i":/go-cover-dir "$i" + GO_COVER_DIR_FOLDERS+=("$i") + fi + done + + if [ ${#GO_COVER_DIR_FOLDERS[@]} -ne 0 ]; then + GO_COVER_DIR_STR=$(printf ",%s" "${GO_COVER_DIR_FOLDERS[@]}") + GO_COVER_DIR_STR=${GO_COVER_DIR_STR:1} + go tool covdata merge -i="${GO_COVER_DIR_STR}" -o go-cover-dir-merged + for i in "${GO_COVER_DIR_FOLDERS[@]}"; do + rm -rf "$i" + done + fi +} + function delete_cluster() { echo "deleting K8s cluster: ${CLUSTER_NAME}" install_tools @@ -269,6 +301,8 @@ if [ "${ACTION}" == "test" ]; then fi make e2e_test exit_on_error "e2e tests failed" + uninstall_yunikorn + copy_go_cover_dir elif [ "${ACTION}" == "install" ]; then check_cmd "${GO}" check_opt "kind-node-image-version" "${CLUSTER_VERSION}"