Bazel remote caching

.github/workflows/bootstrap_bazel_remote_cache.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+echo "::group::Configure Bazel Remote Cache"
+
+if [[ -z "${BAZEL_REMOTE_CACHE_URL}" ]]; then
+	echo "::warning file=.github/workflows/bootstrap_bazel_remote_cache.sh,line=3,endLine=3,title=Running without bazel cache!:: BAZEL_REMOTE_CACHE_URL was not specified, so tests will be doing all the work from scratch."
+else
+	echo "Using bazel remote cache."
+	echo "build --remote_cache=${BAZEL_REMOTE_CACHE_URL}" > .bazelrc
+	echo "test --remote_cache=${BAZEL_REMOTE_CACHE_URL}" > .bazelrc
+fi
+
+echo "::endgroup::"
+
+exit 0

.github/workflows/ci.yml

@@ -97,22 +97,12 @@ jobs:
           large-packages: false
       - name: Checkout code
         uses: actions/checkout@v3
-      # example copied from:
-      # https://github.com/actions/cache/blob/04f198bf0b2a39f7230a4304bf07747a0bddf146/examples.md
-      - name: Cache Bazel
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/bazel
-          key: >
-            ${{ runner.os }}-bazel-${{
-            hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE',
-            'WORKSPACE.bazel', 'MODULE.bazel') }}
-          restore-keys: |
-            ${{ runner.os }}-bazel-
       - name: Presubmit
         run: |
+         ./.github/workflows/bootstrap_bazel_remote_cache.sh
          bazel run //ci:presubmit -- --skip-pulumi-deploy
+        env:
+          BAZEL_REMOTE_CACHE_URL: ${{ secrets.BAZEL_REMOTE_CACHE_URL }}
   Staging:
     # Pulumi doesn't like it when multiple deploys are attempted at once.
     # This is also enforced at the pulumi layer, but i'm sure github actions
@@ -123,7 +113,9 @@ jobs:
       || github.event_name == 'workflow_dispatch'
 
     # Performs all offline testing.
-    needs: Presubmit
+    # remove (maybe temporarily?) the presubmit requirement for this
+    # to speed up testing the cache.
+    # needs: Presubmit
     runs-on: ubuntu-latest
     steps:
       - name: Free Disk Space (Ubuntu)
@@ -135,19 +127,6 @@ jobs:
           docker-images: true
           large-packages: false
           swap-storage: true
-      # example copied from:
-      # https://github.com/actions/cache/blob/04f198bf0b2a39f7230a4304bf07747a0bddf146/examples.md
-      - name: Cache Bazel
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/bazel
-          key: >
-            ${{ runner.os }}-bazel-${{
-            hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE',
-            'WORKSPACE.bazel', 'MODULE.bazel') }}
-          restore-keys: |
-            ${{ runner.os }}-bazel-
         # in order to determine if applying this patch would succeed
         # *on the mainline branch*
         # we have to set the Pulumi state to be the same.
@@ -163,19 +142,26 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # https://github.com/pulumi/pulumi-github/issues/248
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_SECRET }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_SECRETS_CREATOR }}
+          BAZEL_REMOTE_CACHE_URL: ${{ secrets.BAZEL_REMOTE_CACHE_URL }}
       - name: Switch back to candidate branch
         uses: actions/checkout@v3
       - name: Deploy candidate branch to Staging
         # we can run this dirty since the next run will --overwrite anyway
         run: |
+         ./.github/workflows/bootstrap_bazel_remote_cache.sh
          bazel run //ci:presubmit -- \
          --skip-bazel-tests \
          --dangerously-skip-pnpm-lockfile-validation --dirty
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          # https://github.com/pulumi/pulumi-github/issues/248
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_SECRET }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_SECRETS_CREATOR }}
+          BAZEL_REMOTE_CACHE_URL: ${{ secrets.BAZEL_REMOTE_CACHE_URL }}
   Submit:
     concurrency: pulumi_production
     if: github.event_name == 'push'
@@ -193,31 +179,22 @@ jobs:
           large-packages: false
       - name: Checkout code
         uses: actions/checkout@v3
-      # example copied from:
-      # https://github.com/actions/cache/blob/04f198bf0b2a39f7230a4304bf07747a0bddf146/examples.md
-      - name: Cache Bazel
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/bazel
-          key: >
-            ${{ runner.os }}-bazel-${{
-            hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE',
-            'WORKSPACE.bazel', 'MODULE.bazel') }}
-          restore-keys: |
-            ${{ runner.os }}-bazel-
       - name: Submit
         # Use npx to try to generate only
         # bazel generated node_modules
         run: |
+         ./.github/workflows/bootstrap_bazel_remote_cache.sh
           bazel run //ci:submit
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # https://github.com/pulumi/pulumi-github/issues/248
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_SECRET }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_SECRETS_CREATOR }}
+          BAZEL_REMOTE_CACHE_URL: ${{ secrets.BAZEL_REMOTE_CACHE_URL }}
+
   Postsubmit:
     runs-on: ubuntu-latest
     if: github.event_name == 'push'
@@ -234,26 +211,16 @@ jobs:
           swap-storage: true
       - name: Checkout code
         uses: actions/checkout@v3
-      # example copied from:
-      # https://github.com/actions/cache/blob/04f198bf0b2a39f7230a4304bf07747a0bddf146/examples.md
-      - name: Cache Bazel
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/bazel
-          key: >
-            ${{ runner.os }}-bazel-${{
-            hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE',
-            'WORKSPACE.bazel', 'MODULE.bazel') }}
-          restore-keys: |
-            ${{ runner.os }}-bazel-
       - name: Postsubmit
         run: |
+         ./.github/workflows/bootstrap_bazel_remote_cache.sh
           bazel run //ci:postsubmit
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # https://github.com/pulumi/pulumi-github/issues/248
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_SECRET }}
+          BAZEL_REMOTE_CACHE_URL: ${{ secrets.BAZEL_REMOTE_CACHE_URL }}

ci/presubmit.ts

@@ -132,7 +132,39 @@ const cmd = new Command('presubmit')
 		}
 	});
 
+function logError(e: unknown) {
+	if (!(e instanceof Error)) return console.log(e);
+
+	const stack = e.stack ?? Error.prototype.stack;
+
+	if (!stack) return console.log(e);
+
+	const runfilesRoot = process.env['TEST_SRCDIR'];
+
+	if (!runfilesRoot) return console.log(e);
+
+	const runfilesRootIndex = stack.indexOf(runfilesRoot);
+
+	if (runfilesRootIndex == -1) return console.log(e);
+
+	const suffix = stack.slice(runfilesRootIndex + runfilesRoot.length);
+
+	const res = /^([A-Za-z0-9/._]+)\.(?:ts|js):(\d+):(\d+)/.exec(suffix);
+
+	if (res === null) return console.log(e);
+
+	const [, filePrefix, line, offset] = res;
+
+	console.error(
+		WorkflowCommand('error')({
+			file: filePrefix,
+			line: line,
+			col: offset,
+		})('' + e)
+	);
+}
+
 cmd.parseAsync(process.argv).catch(e => {
 	process.exitCode = 2;
-	console.error(e);
+	logError(e);
 });

package.json

@@ -19,8 +19,8 @@
 		"@fortawesome/react-fontawesome": "0.2.0",
 		"@microsoft/api-documenter": "7.22.33",
 		"@microsoft/api-extractor": "7.36.4",
-		"@pulumi/aws": "6.0.4",
-		"@pulumi/awsx": "1.0.5",
+		"@pulumi/aws": "6.6.0",
+		"@pulumi/awsx": "2.0.2",
 		"@pulumi/pulumi": "3.81.0",
 		"@schemastore/package": "0.0.10",
 		"@swc/cli": "0.1.62",
@@ -103,10 +103,14 @@
 		"pnpm": "npm run -- bazel run -- @pnpm//:pnpm --dir $PWD"
 	},
 	"dependencies": {
+		"@aws-sdk/region-config-resolver": "^3.433.0",
 		"@commander-js/extra-typings": "^11.0.0",
 		"@pulumi/command": "4.5.0",
+		"@pulumi/github": "^5.17.0",
+		"@pulumi/random": "^4.13.4",
 		"@react-spring/rafz": "^9.7.3",
 		"@types/bcryptjs": "2.4.3",
+		"aws-sdk": "^2.1478.0",
 		"csstype": "^3.1.1",
 		"devtools-protocol": "^0.0.1193409",
 		"eslint-mdx": "^2.1.0",