Improved integration tests

.github/workflows/base_same_id.yml

@@ -0,0 +1,276 @@
+name: Base same keys
+on:
+  push:
+    branches: [ "main", "actions/base-same-id" ]
+  pull_request:
+    branches: [ "main" ]
+
+
+jobs:
+  run_test:
+    name: Base same id
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - restart: false
+            one_account: true
+            interval: 1m
+            id: 0
+            number_of_providers: 2
+
+          - restart: true
+            one_account: false
+            interval: 2m
+            id: 1
+            number_of_providers: 2
+
+          - restart: false
+            one_account: true
+            interval: 5m
+            id: 2
+            number_of_providers: 2
+
+          - restart: false
+            one_account: true
+            interval: 10m
+            id: 3
+            number_of_providers: 2
+
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+#### prepare binaries
+
+      - name: Download and install binaries
+        run: |
+          /bin/bash download_binaries.sh
+
+      - name: Install binaries
+        run: |
+          sudo mkdir -p /usr/lib/yagna
+          sudo mv golem/downloaded/plugins /usr/lib/yagna/plugins
+          sudo mv golem/downloaded/* /usr/bin/ 
+
+#### Central net router is responsible for local communication of the nodes
+
+      - name: Run central-net router
+        run: |
+          ya-sb-router -l tcp://127.0.0.1:5555&
+          echo CENTRAL_NET_HOST=127.0.0.1:5555 >> $GITHUB_ENV
+
+#### Script for preparing runtime (basic preparation)
+
+      - name: Prepare runtime
+        run: |
+          python advanced/prepare_runtime.py --mode central --payments full --num-providers ${{ matrix.number_of_providers }}
+
+#### Get funding by downloading private keys with assets ready to use
+
+      - name: Download private key
+        run: |
+          set -x
+          curl -sSL -H "Authorization: Bearer ${{ secrets.PRIVATE_KEY_SERVER_KEY }}" https://polygongas.org/keys/get/unused_accounts > downloaded_private_key_0.key
+          echo "Keys left $(curl -sSL -H "Authorization: Bearer ${{ secrets.PRIVATE_KEY_SERVER_KEY }}" https://polygongas.org/keys/count/unused_accounts)"
+          # Check if the downloaded key is a valid private key
+          grep -qE '^[a-fA-F0-9]{64}$' downloaded_private_key_0.key || { echo "Error: File contents do not match expected format: $(cat downloaded_private_key_0.key)"; exit 1; }
+
+#### Provisioning requestor
+
+      - name: Run requestor second account
+        if: matrix.one_account != true
+        run: |
+          cd golem/requestor
+          yagna service run&
+          sleep 4
+          yagna id create --password= --from-private-key=$(cat ../../downloaded_private_key_0.key) identity0
+          yagna app-key create --id identity0 dev_req | sed 's/^/\YAGNA_REQUESTOR_APPKEY=/' >> $GITHUB_ENV
+          echo YAGNA_ACCOUNT_0=$(yagna id show identity0 | grep -oP '(?<=nodeId: )[^\s]+') >> $GITHUB_ENV
+
+      - name: Run requestor one account
+        if: matrix.one_account
+        run: |
+          cd golem/requestor
+          echo "YAGNA_AUTOCONF_ID_SECRET=$(cat ../../downloaded_private_key_0.key)" >> .env
+          yagna service run&
+          sleep 4
+          yagna app-key create dev_req | sed 's/^/\YAGNA_REQUESTOR_APPKEY=/' >> $GITHUB_ENV
+          echo YAGNA_ACCOUNT_0=$(yagna id show | grep -oP '(?<=nodeId: )[^\s]+') >> $GITHUB_ENV
+
+      - name: Restart requestor if needed
+        if: matrix.restart
+        run: |
+          cd golem/requestor
+          yagna service shutdown
+          sleep 4
+          yagna service run&
+          sleep 4
+
+      - name: Set payment interval
+        run: |
+          cd golem/requestor
+          yagna payment process set --account $YAGNA_ACCOUNT_0 --interval ${{ matrix.interval }}
+          yagna payment process info --account $YAGNA_ACCOUNT_0
+
+#### Provisioning providers
+
+      - name: Prepare providers
+        run: |
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            cd golem/provider_$i
+            cat .env
+            golemsp setup --no-interactive
+            echo "\n***** Provider $i - Presets before change: *****\n"
+            ya-provider preset list
+            ya-provider preset update --no-interactive --name=vm --price="cpu=0.0016" --price="duration=0.12"
+            echo "\n***** Provider $i - Presets after change: *****\n"
+            ya-provider preset list
+            cd ../..
+          done
+
+      - name: Run providers
+        run: |
+          sudo chown $USER /dev/kvm 
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            cd golem/provider_$i
+            cat .env
+            golemsp setup --no-interactive
+            yagna service run&
+            sleep 6
+            ya-provider run&
+            sleep 4
+            env > yagnadir/env.log
+            cd ../..
+          done
+        env:
+          YA_CONSENT_EXTERNAL: "allow"
+
+      - name: Run base same id test
+        run: |
+          cd examples/base-same-id
+          npm install --silent
+          node base_sameID.js
+
+#### Wait for payments to be finished
+      - name: Wait for payments
+        run: |
+          cd reports
+          npm install --silent
+          node wait_for_payments.js
+        env:
+          YAGNA_DATADIR: ../golem/requestor/yagnadir
+
+#### Stop services
+
+      - name: Stop requestor
+        if: always()
+        run: |
+          cd golem/requestor
+          yagna service shutdown
+
+      - name: Stop providers
+        if: always()
+        run: |
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            cd golem/provider_$i
+            yagna service shutdown
+            cd ../..
+          done
+          sleep 5
+
+#### Report analysis of entities
+
+      - name: Requestor order items
+        run: |
+          cd reports
+          node list_order_items.js
+        env:
+          YAGNA_DATADIR: ../golem/requestor/yagnadir
+
+      - name: Requestor activities and agreements
+        run: |
+          cd reports
+          node list_activities.js
+        env:
+          YAGNA_DATADIR: ../golem/requestor/yagnadir
+
+      - name: Providers activities and agreements
+        run: |
+          cd reports
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            YAGNA_DATADIR=../golem/provider_$i/yagnadir node list_activities.js | tee provider_${i}_activities.log
+          done
+
+      - name: Requestor payments
+        run: |
+          cd reports
+          YAGNA_DATADIR=../golem/requestor/yagnadir node list_payments.js | tee requestor_payments.log
+
+      - name: Providers payments
+        run: |
+          cd reports
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            YAGNA_DATADIR=../golem/provider_$i/yagnadir node list_payments.js | tee provider_${i}_payments.log
+          done
+
+      - name: Requestor debit notes
+        run: |
+          cd reports
+          YAGNA_DATADIR=../golem/requestor/yagnadir node list_debit_notes.js | tee requestor_debit_notes.log
+
+      - name: Providers debit notes
+        run: |
+          cd reports
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            YAGNA_DATADIR=../golem/provider_$i/yagnadir node list_debit_notes.js | tee provider_${i}_debit_notes.log
+          done
+
+      - name: Requestor allocation expenditures
+        run: |
+          cd reports
+          YAGNA_DATADIR=../golem/requestor/yagnadir node list_expenditures.js | tee requestor_expenditures.log
+
+#### Prepare logs by removing binary images and pack all logs and db together
+
+      - name: Prepare logs
+        if: always()
+        run: |
+          # remove image cache from upload
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            rm -r golem/provider_$i/providerdir/exe-unit/cache
+          done  
+          tar -cf data_${{ matrix.id }}.tar golem/requestor/yagnadir
+
+          # append provider files
+          for i in $(seq 0 $((${{ matrix.number_of_providers }} - 1))); do
+            tar -rf data_${{ matrix.id }}.tar golem/provider_$i/yagnadir
+            tar -rf data_${{ matrix.id }}.tar golem/provider_$i/providerdir
+          done
+          
+          # append reports
+          tar -rf data_${{ matrix.id }}.tar reports/*.log || true
+          
+          xz data_${{ matrix.id }}.tar
+
+#### Upload logs
+
+      - name: Upload logs - Set current date
+        if: always()
+        run: echo "CURRENT_DATE_FOR_UPLOAD=$(date -u +"%Y-%m-%dT%H%M%SZ")" >> $GITHUB_ENV
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: data_${{ matrix.id }}_${{env.CURRENT_DATE_FOR_UPLOAD}}.tar.xz
+          path: data_${{ matrix.id }}.tar.xz
+
+#### Upload used private key back to the server
+
+      - name: Upload used private key back to the server
+        if: always()
+        run: |
+          grep -qE '^[a-fA-F0-9]{64}$' downloaded_private_key_0.key || { echo "Error: File contents do not match expected format: $(cat downloaded_private_key_0.key)"; exit 1; }
+          curl -sSL -H "Authorization: Bearer ${{ secrets.PRIVATE_KEY_SERVER_KEY }}" -d $(cat downloaded_private_key_0.key) https://polygongas.org/keys/add/used_accounts