From e70aa78c38cf3fa0e7b29656acbd085473dbda4c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 31 Oct 2023 13:01:00 -0400 Subject: [PATCH] Onboard jenkins prod docker images to github actions (#507) * Onboard jenkins prod docker images to github actions Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu * Add more Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- .github/workflows/build.yml | 102 ++++++++++++------ .github/workflows/dco-check.yml | 18 ---- .../workflows/multi-node-test-workflow.yml | 20 +++- 3 files changed, 88 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/dco-check.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60cf75d1..9c26cda3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,13 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + linux-build: + needs: Get-CI-Image-Tag strategy: matrix: java: @@ -18,7 +24,16 @@ jobs: # Job name name: Build Asynchronous Search # This job runs on Linux. + outputs: + build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }} runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} @@ -29,33 +44,68 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Build with Gradle - run: ./gradlew build + id: step-build-test-linux + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew build" + plugin=`basename $(ls build/distributions/*.zip)` + echo plugin $plugin + mv -v build/distributions/$plugin ./ + echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT + - name: Uploads coverage + uses: codecov/codecov-action@v1.2.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + - uses: actions/upload-artifact@v3 + with: + name: async-plugin-linux-${{ matrix.java }} + path: ${{ steps.step-build-test-linux.outputs.build-test-linux }} + if-no-files-found: error + + linux-test-docker: + needs: linux-build + strategy: + matrix: + java: + - 11 + - 17 + # Job name + name: Test Asynchronous Search with opensearchstaging docker + # This job runs on Linux. + runs-on: ubuntu-latest + steps: + - name: Checkout Branch + uses: actions/checkout@v2 + - uses: actions/download-artifact@v3 + with: + name: async-plugin-linux-${{ matrix.java }} - name: Pull and Run Docker for security tests run: | - version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'` - IFS='-' read -r -a version_array <<< "$version" - plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done - docker_version="${version_array[0]}-${version_array[1]}" + plugin=${{ needs.linux-build.outputs.build-test-linux }} + version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` + plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` + qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - echo version $version - echo plugin_version $plugin_version - echo docker_version $docker_version + if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then + qualifier=-${qualifier} + else + qualifier="" + fi - pwd=`pwd` - echo $pwd - cd .. - pwd1=`pwd` - echo $pwd1 - list_of_all_files=`ls asynchronous-search/build/distributions/` - echo "listing distributions" - echo $list_of_all_files + docker_version=$version$qualifier + + + echo plugin version plugin_version qualifier docker_version + echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" + + pwd && ls -l ./$plugin if docker pull opensearchstaging/opensearch:$docker_version then echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-asynchronous-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-asynchronous-search; fi" >> Dockerfile - echo "ADD asynchronous-search/build/distributions/opensearch-asynchronous-search-$plugin_version.zip /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-asynchronous-search-$plugin_version.zip" >> Dockerfile + echo "ADD $plugin /tmp/" >> Dockerfile + echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile docker build -t opensearch-asynchronous-search:test . echo "imagePresent=true" >> $GITHUB_ENV else @@ -85,20 +135,7 @@ jobs: with: name: logs path: build/testclusters/integTest-*/logs/* - - name: Create Artifact Path - run: | - mkdir -p asynchronous-search-artifacts - cp ./build/distributions/*.zip asynchronous-search-artifacts - - name: Uploads coverage - uses: codecov/codecov-action@v1.2.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - - name: Upload Artifacts - uses: actions/upload-artifact@v1 - with: - name: asynchronous-search-plugin-ubuntu - path: asynchronous-search-artifacts + windows-build: # Job name name: Build Asynchronous Search @@ -127,6 +164,7 @@ jobs: with: name: asynchronous-search-plugin-windows path: asynchronous-search-artifacts + mac-os-build: # Job name name: Build Asynchronous Search diff --git a/.github/workflows/dco-check.yml b/.github/workflows/dco-check.yml deleted file mode 100644 index cf30ea89..00000000 --- a/.github/workflows/dco-check.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 60bfa30b..6d07db88 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -12,11 +12,24 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + build: # Job name + needs: Get-CI-Image-Tag name: Build Asynchronous Search # This job runs on Linux runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK 11 @@ -27,11 +40,14 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Run integration tests with multi node config - run: ./gradlew integTest -PnumNodes=5 + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTest -PnumNodes=5" - name: Run Backwards Compatibility Tests run: | echo "Running backwards compatibility tests ..." - ./gradlew bwcTestSuite -Dtests.security.manager=false + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false" - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure()