From b9f699f3a560076a6d307f0f6f8221a193116eca Mon Sep 17 00:00:00 2001 From: pSchlarb Date: Wed, 1 Mar 2023 10:17:27 +0000 Subject: [PATCH 1/3] rocksdb circ dep fix Signed-off-by: pSchlarb --- .github/workflows/Push.yaml | 25 +++++----- .../workflows/build/Dockerfile.ubuntu-2004 | 50 +++---------------- .github/workflows/reuseable_test.yaml | 25 +++++++++- setup.py | 2 +- 4 files changed, 45 insertions(+), 57 deletions(-) diff --git a/.github/workflows/Push.yaml b/.github/workflows/Push.yaml index 5db35420bc..013db8e22d 100644 --- a/.github/workflows/Push.yaml +++ b/.github/workflows/Push.yaml @@ -3,6 +3,7 @@ on: push: branches: - ubuntu-20.04-upgrade + - rocksdbfix paths: - '**.py' - '.github/**' @@ -42,29 +43,29 @@ jobs: CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - - plenum_tests: - name: Test Plenum - needs: [workflow-setup, lint, build-docker-image] - uses: ./.github/workflows/reuseable_test.yaml - with: - GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} - UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - + build_packages: name: Build Packages - needs: [workflow-setup, plenum_tests] + needs: [workflow-setup, build-docker-image] uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 with: DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} isDev: 'true' isRC: 'false' - moduleName: plenum + moduleName: plenum + plenum_tests: + name: Test Plenum + needs: [workflow-setup, lint, build_packages] + uses: ./.github/workflows/reuseable_test.yaml + with: + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + publish_artifacts: name: Publish Artifacts - needs: [workflow-setup, build_packages] + needs: [workflow-setup, plenum_tests] if: needs.workflow-setup.outputs.publish == 'true' uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v1 with: diff --git a/.github/workflows/build/Dockerfile.ubuntu-2004 b/.github/workflows/build/Dockerfile.ubuntu-2004 index 119806eef4..86becbcd08 100644 --- a/.github/workflows/build/Dockerfile.ubuntu-2004 +++ b/.github/workflows/build/Dockerfile.ubuntu-2004 @@ -6,13 +6,12 @@ ARG user=indy RUN apt-get update -y && apt-get install -y \ # common stuff git \ - wget \ - gnupg \ apt-transport-https \ - ca-certificates \ - apt-utils \ - curl \ - jq + apt-utils\ + wget\ + curl\ + jq\ + gnupg # ======================================================================================================== # Update repository signing keys @@ -31,45 +30,10 @@ RUN echo "deb https://hyperledger.jfrog.io/artifactory/indy focal dev" >> /etc/ echo "deb https://repo.sovrin.org/deb bionic master" >> /etc/apt/sources.list && \ echo "deb https://repo.sovrin.org/sdk/deb bionic master" >> /etc/apt/sources.list -# Sovrin -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 RUN apt-get update -y && apt-get install -y \ # Python python3-pip \ - python3-nacl \ - # rocksdb python wrapper - rocksdb=5.8.8 \ - libgflags-dev \ - libsnappy-dev \ - zlib1g-dev \ - libbz2-dev \ - liblz4-dev \ - libgflags-dev \ - # zstd is needed for caching in github actions pipeline - zstd \ - # fpm - ruby \ - ruby-dev \ - rubygems \ - gcc \ - make \ - # Indy Node and Plenum - libssl1.0.0 \ - ursa=0.3.2-1 \ - # Indy SDK - libindy=1.15.0~1625-bionic \ - # Need to move libursa.so to parent dir - && mv /usr/lib/ursa/* /usr/lib && rm -rf /usr/lib/ursa + rubygems &&\ + gem install fpm -RUN pip3 install -U \ - # Required by setup.py - setuptools==50.3.2 - -# install rake -RUN gem install --no-document rake -## install fpm; needs to be pinned to 1.13.1 because some packages cannot be built with the newest release 1.14.0 -RUN gem install --no-document fpm -v 1.14.2 - -RUN apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/reuseable_test.yaml b/.github/workflows/reuseable_test.yaml index cbc77d0901..694e186d4f 100644 --- a/.github/workflows/reuseable_test.yaml +++ b/.github/workflows/reuseable_test.yaml @@ -44,6 +44,18 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + - name: Download plenum deb build + uses: actions/download-artifact@v3 + with: + name: plenum-deb + + - name: Install dependencies from deb + run: | + ls -lisha + aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") + apt install ./indy-plenum_*.deb $aptStr libindy + ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so + - name: Install dependencies run: | # Explicitly use the existing pip cache location in the plenum-build image. @@ -98,11 +110,22 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + - name: Download plenum deb build + uses: actions/download-artifact@v3 + with: + name: plenum-deb + + - name: Install dependencies from deb + run: | + ls -lisha + aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") + apt install ./indy-plenum_*.deb $aptStr libindy + ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so + - name: Install dependencies run: | # Explicitly use the existing pip cache location in the plenum-build image. pip --cache-dir /root/.cache/pip install .[tests] - - name: Run Indy Plenum ${{ matrix.module }} tests run: python3 -m pytest -l -vv --junitxml=test-result-plenum-${{ matrix.module }}.xml ${{ matrix.module }} diff --git a/setup.py b/setup.py index f2251f5d4e..ce43dc32c5 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ def run(self): 'packaging', # 'portalocker==2.2.1', 'portalocker', - 'prompt_toolkit>=3.0.16', + 'prompt_toolkit>=3.0.18', # 'psutil==5.6.6', 'psutil', # Pinned because tests fail with v.0.9 From 029a7eeecb2448534a96d8a6f97b99291eea160a Mon Sep 17 00:00:00 2001 From: pSchlarb Date: Thu, 2 Mar 2023 08:27:53 +0000 Subject: [PATCH 2/3] missing rocksdb deps Signed-off-by: pSchlarb --- .github/workflows/build/Dockerfile.ubuntu-2004 | 2 +- .github/workflows/reuseable_test.yaml | 4 ++-- build-scripts/ubuntu-2004/build-3rd-parties.sh | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build/Dockerfile.ubuntu-2004 b/.github/workflows/build/Dockerfile.ubuntu-2004 index 86becbcd08..25fb6888d0 100644 --- a/.github/workflows/build/Dockerfile.ubuntu-2004 +++ b/.github/workflows/build/Dockerfile.ubuntu-2004 @@ -35,5 +35,5 @@ RUN apt-get update -y && apt-get install -y \ # Python python3-pip \ rubygems &&\ - gem install fpm + gem install --no-document fpm -v 1.14.2 diff --git a/.github/workflows/reuseable_test.yaml b/.github/workflows/reuseable_test.yaml index 694e186d4f..ed16bf6a43 100644 --- a/.github/workflows/reuseable_test.yaml +++ b/.github/workflows/reuseable_test.yaml @@ -53,7 +53,7 @@ jobs: run: | ls -lisha aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") - apt install ./indy-plenum_*.deb $aptStr libindy + apt install -y ./indy-plenum_*.deb $aptStr libindy ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so - name: Install dependencies @@ -119,7 +119,7 @@ jobs: run: | ls -lisha aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") - apt install ./indy-plenum_*.deb $aptStr libindy + apt install -y ./indy-plenum_*.deb $aptStr libindy ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so - name: Install dependencies diff --git a/build-scripts/ubuntu-2004/build-3rd-parties.sh b/build-scripts/ubuntu-2004/build-3rd-parties.sh index 240ae23d2e..de68f4bdec 100755 --- a/build-scripts/ubuntu-2004/build-3rd-parties.sh +++ b/build-scripts/ubuntu-2004/build-3rd-parties.sh @@ -8,7 +8,8 @@ OUTPUT_PATH=${1:-.} function build_rocksdb_deb { VERSION=$1 VERSION_TAG="rocksdb-$VERSION" - + #Install rocksdb requirements (libsnappy lbz2 llz4) + apt update && apt install -y libsnappy-dev libbz2-dev liblz4-dev zlib1g-dev libgflags-dev git clone https://github.com/evernym/rocksdb.git /tmp/rocksdb scriptpath="$(dirname "$(realpath "$0")")"/make_rocksdb.sh cd /tmp/rocksdb From 99d020b02efd89b54f401c5b8b9f357a1cf15229 Mon Sep 17 00:00:00 2001 From: pSchlarb Date: Thu, 2 Mar 2023 14:03:12 +0000 Subject: [PATCH 3/3] changed ordering of pipeline tasks Signed-off-by: pSchlarb --- .github/workflows/PR.yaml | 20 ++++++++++---------- .github/workflows/releasepr.yaml | 22 +++++++++++----------- .github/workflows/reuseable_test.yaml | 2 -- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 0a599e7cfd..333a0a21f6 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -55,17 +55,9 @@ jobs: DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - plenum_tests: - name: Test Plenum - needs: [workflow-setup, lint, build-docker-image] - uses: ./.github/workflows/reuseable_test.yaml - with: - GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} - UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} - build_packages: name: Build Packages - needs: [workflow-setup, plenum_tests] + needs: [workflow-setup, build-docker-image] uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 with: DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} @@ -74,10 +66,18 @@ jobs: isRC: 'false' moduleName: plenum + plenum_tests: + name: Test Plenum + needs: [workflow-setup, lint, build_packages] + uses: ./.github/workflows/reuseable_test.yaml + with: + GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} + statusCheck: name: statusCheck runs-on: ubuntu-latest - needs: [workflow-setup, build_packages] + needs: [workflow-setup, plenum_tests] if: ${{ needs.workflow-setup.outputs.testsNeeded == 'false' || success() }} steps: - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." ' \ No newline at end of file diff --git a/.github/workflows/releasepr.yaml b/.github/workflows/releasepr.yaml index 4427434dc4..c4f04e8521 100644 --- a/.github/workflows/releasepr.yaml +++ b/.github/workflows/releasepr.yaml @@ -49,18 +49,9 @@ jobs: DOCKER_IMAGE: ghcr.io/${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }} - plenum_tests: - name: Test Plenum - needs: [release-infos, lint, build-docker-image] - if: needs.release-infos.outputs.isVersionBump == 'true' - uses: ./.github/workflows/reuseable_test.yaml - with: - GITHUB_REPOSITORY_NAME: ${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }} - UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }} - build_packages: name: Build Packages - needs: [release-infos, plenum_tests] + needs: [release-infos, build-docker-image] if: needs.release-infos.outputs.isVersionBump == 'true' uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 with: @@ -70,9 +61,18 @@ jobs: isRC: '${{ needs.release-infos.outputs.isPreRelease }}' moduleName: plenum + plenum_tests: + name: Test Plenum + needs: [release-infos, lint, build_packages] + if: needs.release-infos.outputs.isVersionBump == 'true' + uses: ./.github/workflows/reuseable_test.yaml + with: + GITHUB_REPOSITORY_NAME: ${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }} + UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }} + statusCheck: name: statusCheck runs-on: ubuntu-latest - needs: [build_packages] + needs: [plenum_tests] steps: - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." ' \ No newline at end of file diff --git a/.github/workflows/reuseable_test.yaml b/.github/workflows/reuseable_test.yaml index ed16bf6a43..f9bc1f8c74 100644 --- a/.github/workflows/reuseable_test.yaml +++ b/.github/workflows/reuseable_test.yaml @@ -51,7 +51,6 @@ jobs: - name: Install dependencies from deb run: | - ls -lisha aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") apt install -y ./indy-plenum_*.deb $aptStr libindy ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so @@ -117,7 +116,6 @@ jobs: - name: Install dependencies from deb run: | - ls -lisha aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n") apt install -y ./indy-plenum_*.deb $aptStr libindy ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so