Merge pull request #1635 from pSchlarb/rocksdbfix

Rocksdbfix
hyperledger · Mar 3, 2023 · 14ced3d · 14ced3d
2 parents 48c9d8e + 99d020b
commit 14ced3d
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 79 deletions.
diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml
@@ -55,17 +55,9 @@ jobs:
       DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build
       UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
 
-  plenum_tests:
-    name: Test Plenum
-    needs: [workflow-setup, lint, build-docker-image]
-    uses: ./.github/workflows/reuseable_test.yaml
-    with:
-      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
-      UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
-
   build_packages:
     name: Build Packages
-    needs: [workflow-setup, plenum_tests]
+    needs: [workflow-setup, build-docker-image]
     uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1
     with:
       DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
@@ -74,10 +66,18 @@ jobs:
       isRC: 'false'
       moduleName: plenum
 
+  plenum_tests:
+    name: Test Plenum
+    needs: [workflow-setup, lint, build_packages]
+    uses: ./.github/workflows/reuseable_test.yaml
+    with:
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+      UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
+
   statusCheck:
     name: statusCheck
     runs-on: ubuntu-latest
-    needs: [workflow-setup, build_packages]
+    needs: [workflow-setup, plenum_tests]
     if: ${{ needs.workflow-setup.outputs.testsNeeded == 'false' || success() }}
     steps:
       - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." '
diff --git a/.github/workflows/Push.yaml b/.github/workflows/Push.yaml
@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - ubuntu-20.04-upgrade
+      - rocksdbfix
     paths:
       - '**.py'
       - '.github/**'
@@ -42,29 +43,29 @@ jobs:
       CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }}
       DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build
       UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
-
-  plenum_tests:
-    name: Test Plenum
-    needs: [workflow-setup, lint, build-docker-image]
-    uses: ./.github/workflows/reuseable_test.yaml
-    with:
-      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
-      UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
-
+
   build_packages:
     name: Build Packages
-    needs: [workflow-setup, plenum_tests]
+    needs: [workflow-setup, build-docker-image]
     uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1
     with:
       DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
       UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
       isDev: 'true'
       isRC: 'false'
-      moduleName: plenum
+      moduleName: plenum      
 
+  plenum_tests:
+    name: Test Plenum
+    needs: [workflow-setup, lint, build_packages]
+    uses: ./.github/workflows/reuseable_test.yaml
+    with:
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+      UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }}
+
   publish_artifacts:
     name: Publish Artifacts
-    needs: [workflow-setup, build_packages]
+    needs: [workflow-setup, plenum_tests]
     if: needs.workflow-setup.outputs.publish == 'true'
     uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v1
     with:

diff --git a/.github/workflows/build/Dockerfile.ubuntu-2004 b/.github/workflows/build/Dockerfile.ubuntu-2004
@@ -6,13 +6,12 @@ ARG user=indy
 RUN apt-get update -y && apt-get install -y \
     # common stuff
     git \
-    wget \
-    gnupg \
     apt-transport-https \
-    ca-certificates \
-    apt-utils \
-    curl \
-    jq
+    apt-utils\
+    wget\
+    curl\
+    jq\
+    gnupg
 
 # ========================================================================================================
 # Update repository signing keys
@@ -31,45 +30,10 @@ RUN echo "deb https://hyperledger.jfrog.io/artifactory/indy focal dev"  >> /etc/
     echo "deb https://repo.sovrin.org/deb bionic master" >> /etc/apt/sources.list && \
     echo "deb https://repo.sovrin.org/sdk/deb bionic master" >> /etc/apt/sources.list
 
-# Sovrin
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 
 
 RUN apt-get update -y && apt-get install -y \
     # Python
     python3-pip \
-    python3-nacl \
-    # rocksdb python wrapper
-    rocksdb=5.8.8 \
-    libgflags-dev \
-    libsnappy-dev \
-    zlib1g-dev \
-    libbz2-dev \
-    liblz4-dev \
-    libgflags-dev \
-    # zstd is needed for caching in github actions pipeline
-    zstd \
-    # fpm
-    ruby \
-    ruby-dev \
-    rubygems \
-    gcc \
-    make \
-    # Indy Node and Plenum
-    libssl1.0.0 \
-    ursa=0.3.2-1 \
-    # Indy SDK
-    libindy=1.15.0~1625-bionic \
-    # Need to move libursa.so to parent dir
-    && mv /usr/lib/ursa/* /usr/lib && rm -rf /usr/lib/ursa
+    rubygems &&\
+    gem install --no-document fpm -v 1.14.2
 
-RUN pip3 install -U \
-    # Required by setup.py
-    setuptools==50.3.2
-
-# install rake
-RUN gem install --no-document rake 
-## install fpm; needs to be pinned to 1.13.1 because some packages cannot be built with the newest release 1.14.0
-RUN gem install --no-document fpm -v 1.14.2
-
-RUN apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
diff --git a/.github/workflows/releasepr.yaml b/.github/workflows/releasepr.yaml
@@ -49,18 +49,9 @@ jobs:
       DOCKER_IMAGE: ghcr.io/${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build
       UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }}
 
-  plenum_tests:
-    name: Test Plenum
-    needs: [release-infos, lint, build-docker-image]
-    if: needs.release-infos.outputs.isVersionBump == 'true'
-    uses: ./.github/workflows/reuseable_test.yaml
-    with:
-      GITHUB_REPOSITORY_NAME: ${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}
-      UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }}
-
   build_packages:
     name: Build Packages
-    needs: [release-infos, plenum_tests]
+    needs: [release-infos, build-docker-image]
     if: needs.release-infos.outputs.isVersionBump == 'true'
     uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1
     with:
@@ -70,9 +61,18 @@ jobs:
       isRC: '${{ needs.release-infos.outputs.isPreRelease }}'
       moduleName: plenum
 
+  plenum_tests:
+    name: Test Plenum
+    needs: [release-infos, lint, build_packages]
+    if: needs.release-infos.outputs.isVersionBump == 'true'
+    uses: ./.github/workflows/reuseable_test.yaml
+    with:
+      GITHUB_REPOSITORY_NAME: ${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}
+      UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }}
+
   statusCheck:
     name: statusCheck
     runs-on: ubuntu-latest
-    needs: [build_packages]
+    needs: [plenum_tests]
     steps:
       - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." '
diff --git a/.github/workflows/reuseable_test.yaml b/.github/workflows/reuseable_test.yaml
@@ -44,6 +44,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Download plenum deb build
+        uses: actions/download-artifact@v3
+        with:
+          name: plenum-deb
+
+      - name: Install dependencies from deb
+        run: |
+          aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n")
+          apt install -y ./indy-plenum_*.deb $aptStr libindy
+          ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so
+
       - name: Install dependencies
         run: |
           # Explicitly use the existing pip cache location in the plenum-build image.
@@ -98,11 +109,21 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Download plenum deb build
+        uses: actions/download-artifact@v3
+        with:
+          name: plenum-deb
+
+      - name: Install dependencies from deb
+        run: |
+          aptStr=$(dpkg-deb -I indy-plenum_*.deb | grep -P "Depends: " | sed 's~Depends:~~g;s~(~~g;s~)~~g;s~>~~g;s~<~~g' | tr -d " \t\n\r" | tr "," "\n")
+          apt install -y ./indy-plenum_*.deb $aptStr libindy
+          ln -s /usr/lib/ursa/libursa.so /usr/lib/libursa.so
+
       - name: Install dependencies
         run: |
           # Explicitly use the existing pip cache location in the plenum-build image.
           pip --cache-dir /root/.cache/pip install .[tests]
-         
 
       - name: Run Indy Plenum ${{ matrix.module }} tests
         run: python3 -m pytest -l -vv --junitxml=test-result-plenum-${{ matrix.module }}.xml ${{ matrix.module }}

diff --git a/build-scripts/ubuntu-2004/build-3rd-parties.sh b/build-scripts/ubuntu-2004/build-3rd-parties.sh
@@ -8,7 +8,8 @@ OUTPUT_PATH=${1:-.}
 function build_rocksdb_deb {
     VERSION=$1
     VERSION_TAG="rocksdb-$VERSION"
-
+    #Install rocksdb requirements (libsnappy lbz2 llz4)
+    apt update && apt install -y libsnappy-dev libbz2-dev liblz4-dev zlib1g-dev libgflags-dev
     git clone https://github.com/evernym/rocksdb.git /tmp/rocksdb
     scriptpath="$(dirname "$(realpath "$0")")"/make_rocksdb.sh
     cd /tmp/rocksdb

diff --git a/setup.py b/setup.py
@@ -115,7 +115,7 @@ def run(self):
                         'packaging',
                         # 'portalocker==2.2.1',
                         'portalocker',
-                        'prompt_toolkit>=3.0.16',
+                        'prompt_toolkit>=3.0.18',
                         # 'psutil==5.6.6',
                         'psutil',
                         # Pinned because tests fail with v.0.9