From 3bf013f0b016792058f3175f9119a2312ce0a99e Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 12 Oct 2022 16:57:15 +0200 Subject: [PATCH] feat: add support for gateway race configuration via wrangler secret (#100) Co-authored-by: Alan Shaw --- packages/edge-gateway/README.md | 2 + packages/edge-gateway/src/constants.js | 9 +++++ packages/edge-gateway/src/env.js | 35 ++++++++++++++++- packages/edge-gateway/src/version.js | 4 +- packages/edge-gateway/test/env.spec.js | 38 +++++++++++++++++++ packages/edge-gateway/test/utils/miniflare.js | 6 +++ packages/edge-gateway/test/version.spec.js | 6 ++- packages/edge-gateway/wrangler.toml | 6 --- 8 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 packages/edge-gateway/test/env.spec.js diff --git a/packages/edge-gateway/README.md b/packages/edge-gateway/README.md index 0bc6b1b..6bf318e 100644 --- a/packages/edge-gateway/README.md +++ b/packages/edge-gateway/README.md @@ -15,6 +15,8 @@ wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry wrangler secret put LOKI_URL --env $(whoami) # Get from Loki wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki + wrangler secret put IPFS_GATEWAYS_RACE_L1 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://ipfs.io\"]) + wrangler secret put IPFS_GATEWAYS_RACE_L2 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]) ``` - `pnpm run publish` - Publish the worker under desired env. An alias for `wrangler publish --env $(whoami)` diff --git a/packages/edge-gateway/src/constants.js b/packages/edge-gateway/src/constants.js index d7646bd..058079b 100644 --- a/packages/edge-gateway/src/constants.js +++ b/packages/edge-gateway/src/constants.js @@ -13,3 +13,12 @@ export const RESOLUTION_IDENTIFIERS = { CACHE_ZONE: 'cache-zone', PERMA_CACHE: 'perma-cache' } + +export const DEFAULT_RACE_L1_GATEWAYS = [ + 'https://ipfs.io' +] + +export const DEFAULT_RACE_L2_GATEWAYS = [ + 'https://cf.dag.haus', + 'https://w3link.mypinata.cloud' +] diff --git a/packages/edge-gateway/src/env.js b/packages/edge-gateway/src/env.js index 494d062..fedaa39 100644 --- a/packages/edge-gateway/src/env.js +++ b/packages/edge-gateway/src/env.js @@ -5,6 +5,10 @@ import { Logging } from '@web3-storage/worker-utils/loki' import { createGatewayRacer } from 'ipfs-gateway-race' import pkg from '../package.json' +import { + DEFAULT_RACE_L1_GATEWAYS, + DEFAULT_RACE_L2_GATEWAYS +} from './constants.js' /** * @typedef {import('./bindings').Env} Env @@ -28,14 +32,19 @@ export function envAll (request, env, ctx) { env.SENTRY_RELEASE = SENTRY_RELEASE env.sentry = getSentry(request, env, ctx) - env.ipfsGatewaysL1 = JSON.parse(env.IPFS_GATEWAYS_RACE_L1) - env.ipfsGatewaysL2 = JSON.parse(env.IPFS_GATEWAYS_RACE_L2) + + // Set Layer 1 racer + env.ipfsGatewaysL1 = parseGatewayUrls(env.IPFS_GATEWAYS_RACE_L1, DEFAULT_RACE_L1_GATEWAYS, env) env.gwRacerL1 = createGatewayRacer(env.ipfsGatewaysL1, { timeout: env.REQUEST_TIMEOUT }) + + // Set Layer 2 racer + env.ipfsGatewaysL2 = parseGatewayUrls(env.IPFS_GATEWAYS_RACE_L2, DEFAULT_RACE_L2_GATEWAYS, env) env.gwRacerL2 = createGatewayRacer(env.ipfsGatewaysL2, { timeout: env.REQUEST_TIMEOUT }) + env.startTime = Date.now() env.isCidVerifierEnabled = env.CID_VERIFIER_ENABLED === 'true' @@ -55,6 +64,28 @@ export function envAll (request, env, ctx) { env.log.time('request') } +/** + * @param {string} input + * @param {string[]} defaultValue + * @param {Env} env + */ +function parseGatewayUrls (input, defaultValue, env) { + let list + try { + list = JSON.parse(input) + // Validate is array and has URLs + if (!Array.isArray(list)) { + throw new Error('invalid gateways list environment variable') + } + list.forEach(gwUrl => new URL(gwUrl)) + } catch (err) { + env.log && env.log.warn(`Invalid JSON string with race Gateways: ${input}`) + list = defaultValue + } + + return list +} + /** * Get sentry instance if configured * diff --git a/packages/edge-gateway/src/version.js b/packages/edge-gateway/src/version.js index deb9544..47b21e3 100644 --- a/packages/edge-gateway/src/version.js +++ b/packages/edge-gateway/src/version.js @@ -10,6 +10,8 @@ export async function versionGet (request, env) { return new JSONResponse({ version: env.VERSION, commit: env.COMMITHASH, - branch: env.BRANCH + branch: env.BRANCH, + raceGatewaysL1: env.ipfsGatewaysL1, + raceGatewaysL2: env.ipfsGatewaysL2 }) } diff --git a/packages/edge-gateway/test/env.spec.js b/packages/edge-gateway/test/env.spec.js new file mode 100644 index 0000000..d5cca74 --- /dev/null +++ b/packages/edge-gateway/test/env.spec.js @@ -0,0 +1,38 @@ +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' +import git from 'git-rev-sync' + +import { + DEFAULT_RACE_L1_GATEWAYS, + DEFAULT_RACE_L2_GATEWAYS +} from '../src/constants.js' +import { test, getMiniflare } from './utils/setup.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const pkg = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8') +) + +// Create a new Miniflare environment for each test +test.before((t) => { + t.context = { + mf: getMiniflare() + } +}) + +test('Defaults to hardcoded gateways if invalid secrets are set', async (t) => { + const mf = getMiniflare({ + IPFS_GATEWAYS_RACE_L1: 'invalid gateways value', + IPFS_GATEWAYS_RACE_L2: '["no-url"]' + }) + + const response = await mf.dispatchFetch('http://localhost:8787/version') + const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json() + + t.is(version, pkg.version) + t.is(commit, git.long(__dirname)) + t.is(branch, git.branch(__dirname)) + t.deepEqual(raceGatewaysL1, DEFAULT_RACE_L1_GATEWAYS) + t.deepEqual(raceGatewaysL2, DEFAULT_RACE_L2_GATEWAYS) +}) diff --git a/packages/edge-gateway/test/utils/miniflare.js b/packages/edge-gateway/test/utils/miniflare.js index f0bddee..965c839 100644 --- a/packages/edge-gateway/test/utils/miniflare.js +++ b/packages/edge-gateway/test/utils/miniflare.js @@ -2,6 +2,11 @@ import fs from 'fs' import path from 'path' import { Miniflare } from 'miniflare' +export const secrets = { + IPFS_GATEWAYS_RACE_L1: '["http://127.0.0.1:9081"]', + IPFS_GATEWAYS_RACE_L2: '["http://localhost:9082", "http://localhost:9083"]' +} + export function getMiniflare (bindings = {}) { let envPath = path.join(process.cwd(), '../../.env') if (!fs.statSync(envPath, { throwIfNoEntry: false })) { @@ -41,6 +46,7 @@ export function getMiniflare (bindings = {}) { PUBLIC_RACE_TTFB: createAnalyticsEngine(), PUBLIC_RACE_STATUS_CODE: createAnalyticsEngine(), REQUEST_TIMEOUT: 3000, + ...secrets, ...bindings } }) diff --git a/packages/edge-gateway/test/version.spec.js b/packages/edge-gateway/test/version.spec.js index 9766243..a18b019 100644 --- a/packages/edge-gateway/test/version.spec.js +++ b/packages/edge-gateway/test/version.spec.js @@ -3,7 +3,7 @@ import path from 'path' import { fileURLToPath } from 'url' import git from 'git-rev-sync' -import { test, getMiniflare } from './utils/setup.js' +import { test, getMiniflare, secrets } from './utils/setup.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const pkg = JSON.parse( @@ -21,9 +21,11 @@ test('Gets Version', async (t) => { const { mf } = t.context const response = await mf.dispatchFetch('http://localhost:8787/version') - const { version, commit, branch } = await response.json() + const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json() t.is(version, pkg.version) t.is(commit, git.long(__dirname)) t.is(branch, git.branch(__dirname)) + t.deepEqual(raceGatewaysL1, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L1)) + t.deepEqual(raceGatewaysL2, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L2)) }) diff --git a/packages/edge-gateway/wrangler.toml b/packages/edge-gateway/wrangler.toml index ac6b4b2..983fcda 100644 --- a/packages/edge-gateway/wrangler.toml +++ b/packages/edge-gateway/wrangler.toml @@ -34,8 +34,6 @@ kv_namespaces = [ ] [env.production.vars] -IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]" -IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]" GATEWAY_HOSTNAME = 'ipfs.dag.haus' CID_VERIFIER_URL = 'https://cid-verifier.dag.haus' EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' @@ -87,8 +85,6 @@ kv_namespaces = [ ] [env.staging.vars] -IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]" -IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]" GATEWAY_HOSTNAME = 'ipfs-staging.dag.haus' CID_VERIFIER_URL = 'https://cid-verifier-staging.dag.haus' EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link' @@ -129,8 +125,6 @@ name = "PUBLIC_RACE_STATUS_CODE" workers_dev = true [env.test.vars] -IPFS_GATEWAYS_RACE_L1 = "[\"http://127.0.0.1:9081\"]" -IPFS_GATEWAYS_RACE_L2 = "[\"http://localhost:9082\", \"http://localhost:9083\"]" GATEWAY_HOSTNAME = 'ipfs.localhost:8787' CID_VERIFIER_URL = 'http://cid-verifier.localhost:8787' EDGE_GATEWAY_API_URL = 'http://localhost:8787'