feat: add support for gateway race configuration via wrangler secret (#…

…100)


Co-authored-by: Alan Shaw <[email protected]>

packages/edge-gateway/README.md

@@ -15,6 +15,8 @@
     wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry
     wrangler secret put LOKI_URL --env $(whoami) # Get from Loki
     wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki
+    wrangler secret put IPFS_GATEWAYS_RACE_L1 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://ipfs.io\"])
+    wrangler secret put IPFS_GATEWAYS_RACE_L2 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"])
   ```
 
 - `pnpm run publish` - Publish the worker under desired env. An alias for `wrangler publish --env $(whoami)`

packages/edge-gateway/src/constants.js

@@ -13,3 +13,12 @@ export const RESOLUTION_IDENTIFIERS = {
   CACHE_ZONE: 'cache-zone',
   PERMA_CACHE: 'perma-cache'
 }
+
+export const DEFAULT_RACE_L1_GATEWAYS = [
+  'https://ipfs.io'
+]
+
+export const DEFAULT_RACE_L2_GATEWAYS = [
+  'https://cf.dag.haus',
+  'https://w3link.mypinata.cloud'
+]

packages/edge-gateway/src/env.js

@@ -5,6 +5,10 @@ import { Logging } from '@web3-storage/worker-utils/loki'
 import { createGatewayRacer } from 'ipfs-gateway-race'
 
 import pkg from '../package.json'
+import {
+  DEFAULT_RACE_L1_GATEWAYS,
+  DEFAULT_RACE_L2_GATEWAYS
+} from './constants.js'
 
 /**
  * @typedef {import('./bindings').Env} Env
@@ -28,14 +32,19 @@ export function envAll (request, env, ctx) {
   env.SENTRY_RELEASE = SENTRY_RELEASE
 
   env.sentry = getSentry(request, env, ctx)
-  env.ipfsGatewaysL1 = JSON.parse(env.IPFS_GATEWAYS_RACE_L1)
-  env.ipfsGatewaysL2 = JSON.parse(env.IPFS_GATEWAYS_RACE_L2)
+
+  // Set Layer 1 racer
+  env.ipfsGatewaysL1 = parseGatewayUrls(env.IPFS_GATEWAYS_RACE_L1, DEFAULT_RACE_L1_GATEWAYS, env)
   env.gwRacerL1 = createGatewayRacer(env.ipfsGatewaysL1, {
     timeout: env.REQUEST_TIMEOUT
   })
+
+  // Set Layer 2 racer
+  env.ipfsGatewaysL2 = parseGatewayUrls(env.IPFS_GATEWAYS_RACE_L2, DEFAULT_RACE_L2_GATEWAYS, env)
   env.gwRacerL2 = createGatewayRacer(env.ipfsGatewaysL2, {
     timeout: env.REQUEST_TIMEOUT
   })
+
   env.startTime = Date.now()
 
   env.isCidVerifierEnabled = env.CID_VERIFIER_ENABLED === 'true'
@@ -55,6 +64,28 @@ export function envAll (request, env, ctx) {
   env.log.time('request')
 }
 
+/**
+ * @param {string} input
+ * @param {string[]} defaultValue
+ * @param {Env} env
+ */
+function parseGatewayUrls (input, defaultValue, env) {
+  let list
+  try {
+    list = JSON.parse(input)
+    // Validate is array and has URLs
+    if (!Array.isArray(list)) {
+      throw new Error('invalid gateways list environment variable')
+    }
+    list.forEach(gwUrl => new URL(gwUrl))
+  } catch (err) {
+    env.log && env.log.warn(`Invalid JSON string with race Gateways: ${input}`)
+    list = defaultValue
+  }
+
+  return list
+}
+
 /**
  * Get sentry instance if configured
  *

packages/edge-gateway/src/version.js

@@ -10,6 +10,8 @@ export async function versionGet (request, env) {
   return new JSONResponse({
     version: env.VERSION,
     commit: env.COMMITHASH,
-    branch: env.BRANCH
+    branch: env.BRANCH,
+    raceGatewaysL1: env.ipfsGatewaysL1,
+    raceGatewaysL2: env.ipfsGatewaysL2
   })
 }

packages/edge-gateway/test/env.spec.js

@@ -0,0 +1,38 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import git from 'git-rev-sync'
+
+import {
+  DEFAULT_RACE_L1_GATEWAYS,
+  DEFAULT_RACE_L2_GATEWAYS
+} from '../src/constants.js'
+import { test, getMiniflare } from './utils/setup.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+// Create a new Miniflare environment for each test
+test.before((t) => {
+  t.context = {
+    mf: getMiniflare()
+  }
+})
+
+test('Defaults to hardcoded gateways if invalid secrets are set', async (t) => {
+  const mf = getMiniflare({
+    IPFS_GATEWAYS_RACE_L1: 'invalid gateways value',
+    IPFS_GATEWAYS_RACE_L2: '["no-url"]'
+  })
+
+  const response = await mf.dispatchFetch('http://localhost:8787/version')
+  const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json()
+
+  t.is(version, pkg.version)
+  t.is(commit, git.long(__dirname))
+  t.is(branch, git.branch(__dirname))
+  t.deepEqual(raceGatewaysL1, DEFAULT_RACE_L1_GATEWAYS)
+  t.deepEqual(raceGatewaysL2, DEFAULT_RACE_L2_GATEWAYS)
+})

packages/edge-gateway/test/utils/miniflare.js

@@ -2,6 +2,11 @@ import fs from 'fs'
 import path from 'path'
 import { Miniflare } from 'miniflare'
 
+export const secrets = {
+  IPFS_GATEWAYS_RACE_L1: '["http://127.0.0.1:9081"]',
+  IPFS_GATEWAYS_RACE_L2: '["http://localhost:9082", "http://localhost:9083"]'
+}
+
 export function getMiniflare (bindings = {}) {
   let envPath = path.join(process.cwd(), '../../.env')
   if (!fs.statSync(envPath, { throwIfNoEntry: false })) {
@@ -41,6 +46,7 @@ export function getMiniflare (bindings = {}) {
       PUBLIC_RACE_TTFB: createAnalyticsEngine(),
       PUBLIC_RACE_STATUS_CODE: createAnalyticsEngine(),
       REQUEST_TIMEOUT: 3000,
+      ...secrets,
       ...bindings
     }
   })

packages/edge-gateway/test/version.spec.js

@@ -3,7 +3,7 @@ import path from 'path'
 import { fileURLToPath } from 'url'
 import git from 'git-rev-sync'
 
-import { test, getMiniflare } from './utils/setup.js'
+import { test, getMiniflare, secrets } from './utils/setup.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const pkg = JSON.parse(
@@ -21,9 +21,11 @@ test('Gets Version', async (t) => {
   const { mf } = t.context
 
   const response = await mf.dispatchFetch('http://localhost:8787/version')
-  const { version, commit, branch } = await response.json()
+  const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json()
 
   t.is(version, pkg.version)
   t.is(commit, git.long(__dirname))
   t.is(branch, git.branch(__dirname))
+  t.deepEqual(raceGatewaysL1, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L1))
+  t.deepEqual(raceGatewaysL2, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L2))
 })

packages/edge-gateway/wrangler.toml

@@ -34,8 +34,6 @@ kv_namespaces = [
 ]
 
 [env.production.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]"
 GATEWAY_HOSTNAME = 'ipfs.dag.haus'
 CID_VERIFIER_URL = 'https://cid-verifier.dag.haus'
 EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link'
@@ -87,8 +85,6 @@ kv_namespaces = [
 ]
 
 [env.staging.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]"
 GATEWAY_HOSTNAME = 'ipfs-staging.dag.haus'
 CID_VERIFIER_URL = 'https://cid-verifier-staging.dag.haus'
 EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link'
@@ -129,8 +125,6 @@ name = "PUBLIC_RACE_STATUS_CODE"
 workers_dev = true
 
 [env.test.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"http://127.0.0.1:9081\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"http://localhost:9082\", \"http://localhost:9083\"]"
 GATEWAY_HOSTNAME = 'ipfs.localhost:8787'
 CID_VERIFIER_URL = 'http://cid-verifier.localhost:8787'
 EDGE_GATEWAY_API_URL = 'http://localhost:8787'