Integration tests (#1716)

# Goal The goal of this PR is: 1. Parallelize the integration tests 2. Enable all remaining (possible) integration tests on Testnet. - Closes #1697 - Closes #1510 - Closes #1509 - Closes #1508 - Closes #1507 - Closes #1506 - Closes #1315 # Discussion This might be a bit much for one PR, but these mostly all interrelated. See inline notes for additional work. # Testing - `make e2e-tests` - Running against Testnet is a bit harder to test, and not all current tests will pass as some things are still not on Testnet
frequency-chain · Oct 17, 2023 · ab1807e · ab1807e
1 parent cbbf507
commit ab1807e
Show file tree

Hide file tree

Showing 56 changed files with 1,111 additions and 972 deletions.
diff --git a/.github/workflows/merge-pr.yml b/.github/workflows/merge-pr.yml
@@ -38,7 +38,7 @@ jobs:
       - name: Set up NodeJs
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
           registry-url: "https://registry.npmjs.org"
           cache-dependency-path: js/api-augment/package-lock.json

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -386,7 +386,7 @@ jobs:
       - name: Set up NodeJs
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
           registry-url: "https://registry.npmjs.org"
           cache-dependency-path: js/api-augment/package-lock.json
@@ -1004,7 +1004,7 @@ jobs:
       - name: Set up NodeJs
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
           registry-url: "https://registry.npmjs.org"
           cache-dependency-path: js/api-augment/package-lock.json

diff --git a/.github/workflows/rococo.yml b/.github/workflows/rococo.yml
@@ -1,5 +1,5 @@
-name: Rococo Integration
-run-name: Rococo Integration Testing ${{github.event.inputs.release-version || github.ref_name}}
+name: Rococo E2E Tests
+run-name: Rococo E2E Testing ${{github.event.inputs.release-version || github.ref_name}}
 concurrency:
   group: ${{github.workflow}}-${{github.ref}}
   cancel-in-progress: true
@@ -14,8 +14,8 @@ env:
   TAG_FROM_UI: ${{github.event.inputs.release-version}}
 
 jobs:
-  run-integration-tests:
-    name: Run Integration Tests
+  run-e2e:
+    name: Run E2E Tests
     runs-on: ubuntu-20.04
     container: ghcr.io/libertydsnp/frequency/ci-base-image
     steps:
@@ -43,14 +43,14 @@ jobs:
         with:
           node-version: 18
           cache: "npm"
-          cache-dependency-path: integration-tests/package-lock.json
+          cache-dependency-path: e2e/package-lock.json
       - name: Install Built api-augment
         run: npm install @frequency-chain/api-augment@${{ env.API_AUGMENT_VERSION }}
-        working-directory: integration-tests
-      - name: Run Integration Tests
-        working-directory: integration-tests
+        working-directory: e2e
+      - name: Run e2e Tests
+        working-directory: e2e
         env:
           CHAIN_ENVIRONMENT: rococo-testnet
           WS_PROVIDER_URL: wss://rpc.rococo.frequency.xyz
-          FUNDING_ACCOUNT_SEED_PHRASE: ${{ secrets.ROCOCO_INTEGRATION_TEST_SEED_PHRASE }}
+          FUNDING_ACCOUNT_SEED_PHRASE: ${{ secrets.ROCOCO_E2E_TEST_SEED_PHRASE }}
         run: npm run test:relay
diff --git a/.github/workflows/verify-pr-commit.yml b/.github/workflows/verify-pr-commit.yml
@@ -21,7 +21,7 @@ jobs:
       rust: ${{steps.filter.outputs.rust}}
       build-binary: ${{steps.filter.outputs.build-binary}}
       cargo-lock: ${{steps.filter.outputs.cargo-lock}}
-      run-integration-tests: ${{steps.filter.outputs.run-integration-tests}}
+      run-e2e: ${{steps.filter.outputs.run-e2e}}
       ci-docker-image: ${{steps.filter.outputs.ci-docker-image}}
     steps:
       - name: Check Out Repo
@@ -44,14 +44,14 @@ jobs:
               - '**/Cargo.toml'
               - '**/Cargo.lock'
               - 'js/api-augment/**'
-              - 'integration-tests/**/*.{ts,json}'
+              - 'e2e/**/*.{ts,json}'
             cargo-lock:
               - '**/Cargo.toml'
               - '**/Cargo.lock'
-            run-integration-tests:
+            run-e2e:
               - '**/*.rs'
               - '**/Cargo.toml'
-              - 'integration-tests/**/*.{ts,json}'
+              - 'e2e/**/*.{ts,json}'
             ci-docker-image:
               - 'tools/ci/docker/ci-base-image.dockerfile'
 
@@ -398,7 +398,7 @@ jobs:
       - name: Set up NodeJs
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
           cache-dependency-path: js/api-augment/package-lock.json
       - name: Install Latest Versions
@@ -630,10 +630,10 @@ jobs:
             must be greater than the latest version on main branch ($spec_version_ref)"
           exit 1
 
-  run-integration-tests:
-    if: needs.changes.outputs.run-integration-tests == 'true'
+  run-e2e:
+    if: needs.changes.outputs.run-e2e == 'true'
     needs: [build-binaries, verify-js-api-augment]
-    name: Run Integration Tests
+    name: Run E2E Tests
     runs-on: ubuntu-20.04
     container: ghcr.io/libertydsnp/frequency/ci-base-image
     steps:
@@ -686,17 +686,17 @@ jobs:
       - name: Set up NodeJs
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
-          cache-dependency-path: integration-tests/package-lock.json
+          cache-dependency-path: e2e/package-lock.json
       - name: Install Built api-augment
         run: npm install ../js/api-augment/dist/frequency-chain-api-augment-0.0.0.tgz
-        working-directory: integration-tests
+        working-directory: e2e
       - name: Install NPM Modules
         run: npm ci
-        working-directory: integration-tests
-      - name: Run Integration Tests
-        working-directory: integration-tests
+        working-directory: e2e
+      - name: Run E2E Tests
+        working-directory: e2e
         env:
           CHAIN_ENVIRONMENT: dev
           WS_PROVIDER_URL: ws://127.0.0.1:9944

diff --git a/Makefile b/Makefile
@@ -81,7 +81,7 @@ lint-audit:
 format-lint: format lint
 
 .PHONY: ci-local
-ci-local: check lint lint-audit test integration-test
+ci-local: check lint lint-audit test e2e
 
 .PHONY: upgrade
 upgrade-local:
@@ -233,23 +233,23 @@ build-mainnet-release:
 test:
 	cargo test --workspace --features runtime-benchmarks,frequency-lint-check
 
-integration-test:
-	./scripts/run_integration_tests.sh
+e2e-tests:
+	./scripts/run_e2e_tests.sh
 
-integration-test-only:
-	./scripts/run_integration_tests.sh -s
+e2e-tests-only:
+	./scripts/run_e2e_tests.sh -s
 
-integration-test-load:
-	./scripts/run_integration_tests.sh load
+e2e-tests-load:
+	./scripts/run_e2e_tests.sh load
 
-integration-test-load-only:
-	./scripts/run_integration_tests.sh -s load
+e2e-tests-load-only:
+	./scripts/run_e2e_tests.sh -s load
 
-integration-test-rococo:
-	./scripts/run_integration_tests.sh -c rococo_testnet
+e2e-tests-rococo:
+	./scripts/run_e2e_tests.sh -c rococo_testnet
 
-integration-test-rococo-local:
-	./scripts/run_integration_tests.sh -c rococo_local
+e2e-tests-rococo-local:
+	./scripts/run_e2e_tests.sh -c rococo_local
 
 .PHONY: try-runtime
 try-runtime:

diff --git a/README.md b/README.md
@@ -34,7 +34,7 @@ Frequency is a [Polkadot](https://www.parity.io/technologies/polkadot) parachain
       - [Stop and Clean Environment](#stop-and-clean-environment)
     - [All in Docker Container](#all-in-docker-container)
   - [Run Tests](#run-tests)
-    - [Integration Tests](#integration-tests)
+    - [E2E Tests](#e2e-tests)
   - [Run Benchmarks](#run-benchmarks)
 - [Format, Lint and Audit Source Code](#format-lint-and-audit-source-code)
 - [Runtime](#runtime)
@@ -276,9 +276,9 @@ make test
 cargo test --features frequency
 ```
 
-### Integration Tests
+### E2E Tests
 
-To run the integration tests, run `make integration-test`.
+To run the end-to-end tests, run `make e2e-tests`.
 
 ## Run Benchmarks
 

diff --git a/integration-tests/.gitignore → e2e/.gitignore b/integration-tests/.gitignore → e2e/.gitignore
diff --git a/integration-tests/.load-test.mocharc.json → e2e/.load-test.mocharc.json b/integration-tests/.load-test.mocharc.json → e2e/.load-test.mocharc.json
@@ -2,7 +2,11 @@
     "exit": true,
     "extension": ["ts"],
     "parallel": false,
-    "require":  ["scaffolding/rootHooks.ts", "scaffolding/extrinsicHelpers.ts"],
+    "require":  [
+        "scaffolding/globalHooks.ts",
+        "scaffolding/rootHooks.ts",
+        "scaffolding/extrinsicHelpers.ts"
+    ],
     "loader": "ts-node/esm",
     "spec": ["./load-tests/**/*.test.ts"],
     "timeout": 60000

diff --git a/e2e/.mocharc.json b/e2e/.mocharc.json
@@ -0,0 +1,13 @@
+{
+    "exit": true,
+    "extension": ["ts"],
+    "parallel": true,
+    "require": [
+        "scaffolding/globalHooks.ts",
+        "scaffolding/rootHooks.ts",
+        "scaffolding/extrinsicHelpers.ts"
+    ],
+    "loader": "ts-node/esm",
+    "spec": ["./{,!(node_modules|load-tests)/**}/*.test.ts"],
+    "timeout": 10000
+}
diff --git a/e2e/.node-version b/e2e/.node-version
@@ -0,0 +1 @@
+18.16.0
diff --git a/integration-tests/.prettierignore → e2e/.prettierignore b/integration-tests/.prettierignore → e2e/.prettierignore
diff --git a/integration-tests/.prettierrc.json → e2e/.prettierrc.json b/integration-tests/.prettierrc.json → e2e/.prettierrc.json
diff --git a/integration-tests/.relay-chain.mocharc.json → e2e/.relay-chain.mocharc.json b/integration-tests/.relay-chain.mocharc.json → e2e/.relay-chain.mocharc.json
@@ -3,14 +3,15 @@
     "extension": [
         "ts"
     ],
-    "parallel": false,
+    "parallel": true,
     "require": [
+        "scaffolding/globalHooks.ts",
         "scaffolding/rootHooks.ts",
         "scaffolding/extrinsicHelpers.ts"
     ],
     "loader": "ts-node/esm",
     "spec": [
-        "./{,!(node_modules|load-tests)/**}/{createMsa,addIPFSMessage,staking,transactions,handles}.test.ts"
+        "./{,!(node_modules|load-tests)/**}/*.test.ts"
     ],
-    "timeout": 250000
+    "timeout": 600000
 }
diff --git a/e2e/.tool-versions b/e2e/.tool-versions
@@ -0,0 +1 @@
+nodejs 18.16.0
diff --git a/integration-tests/README.md → e2e/README.md b/integration-tests/README.md → e2e/README.md
@@ -1,11 +1,11 @@
-Running Integration Tests
+Running E2E Tests
 =========================
 
 To run all tests (and automatically start up a Frequency node):
 
-`make integration-test`
+`make e2e-tests`
 
-To run all tests (after starting up a Frequency node): 
+To run all tests (after starting up a Frequency node):
 
 `npm run test`
 
@@ -17,7 +17,7 @@ Note: this is for the "createMsa" tests
 
 See below for running load tests.
 
-Notes on Integration Testing
+Notes on E2E Testing
 ============================
 
 1. Avoid using anonymous arrow functions for test blocks (`describe`, `it`, `before`, etc).
@@ -79,7 +79,7 @@ The tests in that folder are NOT run with a normal test run.
 It is configured to run in manual sealing mode only. To run the tests, do the following:
 
 ```
-make integration-load-test
+make e2e-tests-load
 ```
 
 That make command does approximately the following:
@@ -91,6 +91,6 @@ make start-manual
 
 2. Run tests
 ```
-cd integration-tests
+cd e2e
 npm run test:load
 ```
diff --git a/...tion-tests/capacity/replenishment.test.ts → e2e/capacity/replenishment.test.ts b/...tion-tests/capacity/replenishment.test.ts → e2e/capacity/replenishment.test.ts
@@ -4,39 +4,41 @@ import { u16, u64 } from "@polkadot/types"
 import assert from "assert";
 import { Extrinsic, ExtrinsicHelper } from "../scaffolding/extrinsicHelpers";
 import {
-  devAccounts,
   createKeys,
   createMsaAndProvider,
   stakeToProvider,
   fundKeypair,
   getNextEpochBlock,
-  TEST_EPOCH_LENGTH,
-  setEpochLength,
   getOrCreateGraphChangeSchema,
   CENTS,
   DOLLARS,
   TokenPerCapacity,
   assertEvent,
   getRemainingCapacity,
-  getNonce
+  getNonce,
 } from "../scaffolding/helpers";
+import { getFundingSource } from "../scaffolding/funding";
+import { isTestnet } from "../scaffolding/env";
 
 describe("Capacity Replenishment Testing: ", function () {
   let schemaId: u16;
+  const fundingSource = getFundingSource("capacity-replenishment");
 
 
   async function createAndStakeProvider(name: string, stakingAmount: bigint): Promise<[KeyringPair, u64]> {
     const stakeKeys = createKeys(name);
-    const stakeProviderId = await createMsaAndProvider(stakeKeys, "ReplProv", 50n * DOLLARS);
+    const stakeProviderId = await createMsaAndProvider(fundingSource, stakeKeys, "ReplProv", 50n * DOLLARS);
     assert.notEqual(stakeProviderId, 0, "stakeProviderId should not be zero");
-    await stakeToProvider(stakeKeys, stakeProviderId, stakingAmount);
+    await stakeToProvider(fundingSource, stakeKeys, stakeProviderId, stakingAmount);
     return [stakeKeys, stakeProviderId];
   }
 
 
   before(async function () {
-    await setEpochLength(devAccounts[0].keys, TEST_EPOCH_LENGTH);
-    schemaId = await getOrCreateGraphChangeSchema();
+    // Replenishment requires the epoch length to be shorter than testnet (set in globalHooks)
+    if (isTestnet()) this.skip();
+
+    schemaId = await getOrCreateGraphChangeSchema(fundingSource);
   });
 
   describe("Capacity is replenished", function () {
@@ -100,8 +102,8 @@ describe("Capacity Replenishment Testing: ", function () {
       const [stakeKeys, stakeProviderId] = await createAndStakeProvider("TinyStake", providerStakeAmt);
       // new user/msa stakes to provider
       const userKeys = createKeys("userKeys");
-      await fundKeypair(devAccounts[0].keys, userKeys, 5n * DOLLARS);
-      let [_, events] = await ExtrinsicHelper.stake(userKeys, stakeProviderId, userStakeAmt).fundAndSend();
+      await fundKeypair(fundingSource, userKeys, 5n * DOLLARS);
+      let [_, events] = await ExtrinsicHelper.stake(userKeys, stakeProviderId, userStakeAmt).fundAndSend(fundingSource);
       assertEvent(events, 'system.ExtrinsicSuccess');
 
       const payload = JSON.stringify({ changeType: 1, fromId: 1, objectId: 2 })
@@ -125,7 +127,7 @@ describe("Capacity Replenishment Testing: ", function () {
       assert(remainingCapacity < callCapacityCost);
 
       // user stakes tiny additional amount
-      [_, events] = await ExtrinsicHelper.stake(userKeys, stakeProviderId, userIncrementAmt).fundAndSend();
+      [_, events] = await ExtrinsicHelper.stake(userKeys, stakeProviderId, userIncrementAmt).fundAndSend(fundingSource);
       assertEvent(events, 'capacity.Staked');
 
       // provider can now send a message
@@ -136,8 +138,8 @@ describe("Capacity Replenishment Testing: ", function () {
       // show that capacity was replenished and then fee deducted.
       let approxExpected = providerStakeAmt + userStakeAmt + userIncrementAmt - callCapacityCost;
       assert(remainingCapacity <= approxExpected, `remainingCapacity = ${remainingCapacity.toString()}`);
-    })
-  })
+    });
+  });
 });
 
 async function drainCapacity(call, stakeProviderId: u64, stakeKeys: KeyringPair): Promise<bigint> {