Workflow caching with nix & faster pre-commit hook (#1929)

* workflow: caching using mk-cache-key.nix and github cache

This allows testing same code only once.

- composite action: .github/actions/setup-workflow-base/action.yml
- caching enalbed workflows:
  - ethereum-contracts
  - sdk-core
  - solidity-semantic-money
  - spec-haskell
  - hot-fuzz
  - automations
  - subgraph

* new pre-commit hook

- remove pre-commit script from all workflows,
- instead, putting workflow filtering in the pre-commit hook.

.github/actions/setup-workflow-base/action.yml

@@ -0,0 +1,52 @@
+name: Setup Workflow Base
+description: >
+  This sets up the base for a workflow, where:
+
+  - nix is installed,
+  - cache is looked up,
+  - and nix dev shell is setup if cache is missed.
+
+inputs:
+  package-name:
+    description: Name of the package where input files are from
+    required: true
+  cache-key-prefix:
+    description: Cache key prefix to attach to the calculated input files hash
+    required: true
+  dev-shell-name:
+    description: Default shell to be used
+
+outputs:
+  cache-hit:
+    value: ${{ steps.cache.outputs.cache-hit }}
+
+runs:
+  using: composite
+  steps:
+    - uses: DeterminateSystems/nix-installer-action@v13
+
+    - name: Load the half-board nix module
+      id: hb
+      run: |
+        ./tasks/mk-cache-key.sh ./packages/${{ inputs.package-name }} > cache.json
+        jq . cache.json
+        key_prefix="${{ inputs.cache-key-prefix }}"
+        path=$(jq '.outputs | join("\n")' cache.json)
+        hash=$(jq -r .hash cache.json)
+        echo "path=$path" >> "$GITHUB_OUTPUT"
+        echo "key=${key_prefix}${hash}" >> "$GITHUB_OUTPUT"
+      shell: nix develop .#mk-cache-key -c bash -xe {0}
+
+    - name: Lookup cache
+      id: cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ fromJSON(steps.hb.outputs.path) }}
+        key: ${{ steps.hb.outputs.key }}
+
+    - name: Initialize nix dev shell
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        node --version
+        yarn --version
+      shell: nix develop .#${{ inputs.dev-shell-name }} -c bash -xe {0}

.github/workflows/call.check-query-schema-against-subgraph.yml

@@ -23,9 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: cachix/install-nix-action@v19
-        with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: DeterminateSystems/nix-installer-action@v13
 
       - name: "Install dependencies"
         run: yarn install --frozen-lockfile

.github/workflows/call.deploy-subgraph.yml

@@ -69,9 +69,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: cachix/install-nix-action@v19
-        with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: DeterminateSystems/nix-installer-action@v13
 
       - name: "Install dependencies"
         run: yarn install --frozen-lockfile

.github/workflows/call.test-automation-contracts.yml

@@ -9,33 +9,44 @@ jobs:
 
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        node-version: [18, 20]
-
     defaults:
       run:
-        shell: nix develop .#ci-node${{ matrix.node-version }} -c bash -xe {0}
+        shell: nix develop .#ci-default -c bash -xe {0}
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: cachix/install-nix-action@v19
+      - name: Setup workflow base
+        id: base
+        uses: ./.github/actions/setup-workflow-base
         with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          package-name: automation-contracts
+          cache-key-prefix: automation-contracts-
+          dev-shell-name: ci-default
 
-      - name: Install, lint and build
+      - name: Install dependencies
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
           yarn install --frozen-lockfile
+          solc --version
+          forge --version
+
+      - name: Lint and build
+        if: steps.base.outputs.cache-hit != 'true'
+        run: |
           yarn lint
           yarn build
 
       - name: Test automation-contracts-scheduler
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
-          echo "FOUNDRY_PROFILE=ci" >> $GITHUB_ENV
           yarn workspace scheduler test
+        env:
+          FOUNDRY_PROFILE: ci
 
       - name: Test automation-contracts-autowrap
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
-          echo "FOUNDRY_PROFILE=ci" >> $GITHUB_ENV
           yarn workspace autowrap test
+        env:
+          FOUNDRY_PROFILE: ci

.github/workflows/call.test-ethereum-contracts.yml

@@ -8,7 +8,7 @@ on:
         type: boolean
 
 jobs:
-  # Ahhhhhhhhhh, it is silly, but the only way to preparing conditional matrix.
+  # When this scrip is written, it is the only way to preparing conditional matrix:
   # Ref: https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
   matrix-prep:
     name: Preparing Conditional Strategy Matrix
@@ -19,16 +19,17 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
 
     steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
 
-    - id: set-matrix
-      run: |
-        if [ "${{ inputs.run-coverage-tests }}" == "true" ];then
-          echo "matrix={\"node-version\": [20]}" >> "$GITHUB_OUTPUT"
-        else
-          echo "matrix={\"node-version\": [18, 20]}" >> "$GITHUB_OUTPUT"
-        fi
+      - name: Set matrix variable
+        id: set-matrix
+        run: |
+          if [ "${{ inputs.run-coverage-tests }}" == "true" ];then
+            echo "matrix={\"node-version\": [20]}" >> "$GITHUB_OUTPUT"
+          else
+            echo "matrix={\"node-version\": [18, 20]}" >> "$GITHUB_OUTPUT"
+          fi
 
   test-ethereum-contracts:
     name: Test ethereum-contracts
@@ -40,41 +41,55 @@ jobs:
     strategy:
       matrix: ${{ fromJson(needs.matrix-prep.outputs.matrix) }}
 
+    env:
+      DEV_SHELL_NAME: ci-node${{ matrix.node-version }}
+
     defaults:
       run:
-        shell: nix develop .#ci-node${{ matrix.node-version }} -c bash -xe {0}
+        shell: nix develop .#${{ env.DEV_SHELL_NAME }} -c bash -xe {0}
 
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - uses: cachix/install-nix-action@v19
-        with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Initialize devShell
+      - name: Set cache key prefix
+        id: set-cache-key-prefix
         run: |
-          node --version
-          yarn --version
-          solc --version
+          node_version=${{ matrix.node-version }}
+          if [ "${{ inputs.run-coverage-tests }}" == false ]; then
+            v=ethereum-contracts-test-${node_version}-
+          else
+            v=ethereum-contracts-coverage-${node_version}-
+          fi
+          echo "cache_key_prefix=$v" >> "$GITHUB_OUTPUT"
+        shell: bash
+
+      - name: Setup workflow base
+        id: base
+        uses: ./.github/actions/setup-workflow-base
+        with:
+          package-name: ethereum-contracts
+          cache-key-prefix: ${{ steps.set-cache-key-prefix.outputs.cache_key_prefix }}
+          dev-shell-name: ${{ env.DEV_SHELL_NAME }}
 
-      - name: Yarn Install
+      - name: Install dependencies
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
           yarn install --frozen-lockfile
+          solc --version
+          forge --version
           npx tsc --version
           npx hardhat --version
 
       - name: Lint and build
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
           yarn lint
           yarn build
 
-      ########################################
-      ## Test Suite
-      ########################################
       - name: Run test suite
-        if: inputs.run-coverage-tests == false
+        if: steps.base.outputs.cache-hit != 'true' && inputs.run-coverage-tests == false
         run: |
           echo "FOUNDRY_PROFILE=ci" >> $GITHUB_ENV
           yarn test
@@ -84,11 +99,8 @@ jobs:
           HARDHAT_TEST_JOBS: 4
           HARDHAT_RUN_PARALLEL: 1
 
-      ########################################
-      ## Coverage Test
-      ########################################
       - name: Run coverage test
-        if: inputs.run-coverage-tests == true
+        if: steps.base.outputs.cache-hit != 'true' && inputs.run-coverage-tests == true
         run: |
           echo "FOUNDRY_PROFILE=ci" >> $GITHUB_ENV
           yarn test-coverage
@@ -105,7 +117,7 @@ jobs:
           HARDHAT_RUN_PARALLEL: 0
 
       - name: Clean up and merge coverage artifacts
-        if: inputs.run-coverage-tests == true
+        if: steps.base.outputs.cache-hit != 'true' && inputs.run-coverage-tests == true
         run: ./tasks/coverage-cleanup.sh
         working-directory: ./packages/ethereum-contracts
 

.github/workflows/call.test-hot-fuzz.yml

@@ -16,23 +16,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: cachix/install-nix-action@v19
+      - name: Setup workflow base
+        id: base
+        uses: ./.github/actions/setup-workflow-base
         with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          package-name: hot-fuzz
+          cache-key-prefix: hot-fuzz-
+          dev-shell-name: ci-hot-fuzz
 
-      - name: Initialize devShell
+      - name: Install dependencies
+        if: steps.base.outputs.cache-hit != 'true'
         run: |
-          set -xe
+          yarn install
+          solc --version
+          forge --version
           slither --version
           echidna --version
 
-      - name: Run Yarn Install
-        run: |
-          yarn install
-
       - name: Run tests
-        run: |
-          cd packages/hot-fuzz
-          ./hot-fuzz contracts/superfluid-tests/SuperHotFuzz.yaml
+        if: steps.base.outputs.cache-hit != 'true'
+        run: ./hot-fuzz contracts/superfluid-tests/SuperHotFuzz.yaml
+        working-directory: packages/hot-fuzz
         env:
-          ECHIDNA_TEST_LIMIT: 10000
+          ECHIDNA_TEST_LIMIT: 20000