Add mix-fetch for nodejs

sdk/typescript/packages/mix-fetch-node/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["@babel/env"]
+}

sdk/typescript/packages/mix-fetch-node/.gitignore

@@ -0,0 +1,2 @@
+src/worker/*.js
+docs

sdk/typescript/packages/mix-fetch-node/README.md

@@ -0,0 +1,14 @@
+# Nym MixFetch
+
+This package is a drop-in replacement for `fetch` in NodeJS to send HTTP requests over the Nym Mixnet.
+
+## Usage
+
+```js
+const { mixFetch } = require('@nymproject/mix-fetch-node');
+
+...
+
+const response = await mixFetch('https://nymtech.net');
+const html = await response.text();
+```

sdk/typescript/packages/mix-fetch-node/internal-dev/index.mjs

@@ -0,0 +1,54 @@
+import { createMixFetch, disconnectMixFetch } from '../dist/cjs/index.js';
+
+/**
+ * The main entry point
+ */
+(async () => {
+  console.log('Tester is starting up...');
+
+  const addr =
+    'D274yd1h3L3pNJzdxE5VgJ7izAsAVMsDrQtFSkKUegfk.8J67cGbcwvrJKF3Kb16HVWWc9AnrFnEibNCm9zCkuVFu@Emswx6KXyjRfq1c2k4d4uD2e6nBSbH1biorCZUei8UNS';
+
+  console.log('About to set up mixFetch...');
+  const { mixFetch } = await createMixFetch({
+    preferredNetworkRequester: addr,
+    clientId: 'node-client1',
+    clientOverride: {
+      coverTraffic: { disableLoopCoverTrafficStream: true },
+      traffic: { disableMainPoissonPacketDistribution: true },
+    },
+    mixFetchOverride: { requestTimeoutMs: 60000 },
+    responseBodyConfigMap: {},
+    extra: {},
+  });
+
+  globalThis.mixFetch = mixFetch;
+
+  if (!globalThis.mixFetch) {
+    console.error('Oh no! Could not create mixFetch');
+  } else {
+    console.log('Ready!');
+  }
+
+  let url = 'https://nymtech.net/.wellknown/network-requester/standard-allowed-list.txt';
+  console.log(`Using mixFetch to get ${url}...`);
+  const args = { mode: 'unsafe-ignore-cors' };
+
+  let resp = await mixFetch(url, args);
+  console.log({ resp });
+  const text = await resp.text();
+
+  console.log('disconnecting');
+  await disconnectMixFetch();
+  console.log('disconnected! all further usages should fail');
+
+  // get an image
+  url = 'https://nymtech.net/favicon.svg';
+  resp = await mixFetch(url, args);
+  console.log({ resp });
+  const buffer = await resp.arrayBuffer();
+  const type = resp.headers.get('Content-Type') || 'image/svg';
+  const blobUrl = URL.createObjectURL(new Blob([buffer], { type }));
+  console.log(JSON.stringify({ bufferBytes: buffer.byteLength, blobUrl }, null, 2));
+  console.log(blobUrl);
+})();

sdk/typescript/packages/mix-fetch-node/jest.config.mjs

@@ -0,0 +1,16 @@
+import preset from 'ts-jest/presets/index.js';
+
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+export default {
+  ...preset.defaults,
+  verbose: true,
+  transform: {
+    '^.+\\.(ts|tsx)$': [
+      'ts-jest',
+      {
+        tsconfig: 'tsconfig.jest.json',
+        useESM: true,
+      },
+    ],
+  },
+};

sdk/typescript/packages/mix-fetch-node/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@nymproject/mix-fetch-node",
+  "version": "1.2.0",
+  "description": "This package is a drop-in replacement for `fetch` in NodeJS to send HTTP requests over the Nym Mixnet.",
+  "license": "Apache-2.0",
+  "author": "Nym Technologies SA",
+  "files": [
+    "dist/cjs/worker.js",
+    "dist/**/*"
+  ],
+  "main": "dist/cjs/index.js",
+  "scripts": {
+    "build": "scripts/build-prod.sh",
+    "build:dev": "scripts/build.sh",
+    "build:worker": "rollup -c rollup-worker.config.mjs",
+    "clean": "rimraf dist",
+    "docs:dev": "run-p docs:watch docs:serve ",
+    "docs:generate": "typedoc",
+    "docs:generate:prod": "typedoc --basePath ./docs/tsdoc/nymproject/sdk/",
+    "docs:prod:build": "scripts/build-prod-docs-collect.sh",
+    "docs:serve": "reload -b -d ./docs -p 3000",
+    "docs:watch": "nodemon --ext ts --watch './src/**/*' --watch './typedoc.json' --exec \"yarn docs:generate\"",
+    "lint": "eslint src",
+    "lint:fix": "eslint src --fix",
+    "start": "tsc -w",
+    "start:dev": "nodemon --watch src -e ts,json --exec 'yarn build:dev:esm'",
+    "test": "node --experimental-fetch --experimental-vm-modules node_modules/jest/bin/jest.js -c=jest.config.mjs --no-cache",
+    "tsc": "tsc --noEmit true"
+  },
+  "dependencies": {
+    "@nymproject/mix-fetch-wasm-node": ">=1.2.0-rc.10 || ^1",
+    "comlink": "^4.3.1",
+    "fake-indexeddb": "^5.0.0",
+    "node-fetch": "^3.3.2",
+    "ws": "^8.14.2"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^24.0.1",
+    "@rollup/plugin-node-resolve": "^15.0.1",
+    "@rollup/plugin-replace": "^5.0.2",
+    "@rollup/plugin-typescript": "^10.0.1",
+    "@rollup/plugin-wasm": "^6.1.1",
+    "@types/jest": "^27.0.1",
+    "@types/node": "^16.7.13",
+    "@typescript-eslint/eslint-plugin": "^5.13.0",
+    "@typescript-eslint/parser": "^5.13.0",
+    "eslint": "^8.10.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-airbnb-typescript": "^16.1.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-import-resolver-root-import": "^1.0.4",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jest": "^26.1.1",
+    "eslint-plugin-jsx-a11y": "^6.5.1",
+    "eslint-plugin-prettier": "^4.0.0",
+    "jest": "^29.5.0",
+    "nodemon": "^2.0.21",
+    "reload": "^3.2.1",
+    "rimraf": "^3.0.2",
+    "rollup": "^3.9.1",
+    "rollup-plugin-base64": "^1.0.1",
+    "rollup-plugin-modify": "^3.0.0",
+    "rollup-plugin-web-worker-loader": "^1.6.1",
+    "ts-jest": "^29.1.0",
+    "ts-loader": "^9.4.2",
+    "typedoc": "^0.24.8",
+    "typescript": "^4.8.4"
+  },
+  "private": false,
+  "types": "./dist/cjs/index.d.ts"
+}

sdk/typescript/packages/mix-fetch-node/rollup-cjs.config.mjs

@@ -0,0 +1,31 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import replace from '@rollup/plugin-replace';
+import resolve from '@rollup/plugin-node-resolve';
+import typescript from '@rollup/plugin-typescript';
+import webWorkerLoader from 'rollup-plugin-web-worker-loader';
+import { wasm } from '@rollup/plugin-wasm';
+
+export default {
+  input: 'src/index.ts',
+  output: {
+    dir: 'dist/cjs',
+    format: 'cjs',
+  },
+  plugins: [
+    webWorkerLoader({ targetPlatform: 'node', inline: false }),
+    replace({
+      values: {
+        "createURLWorkerFactory('web-worker-0.js')":
+          "createURLWorkerFactory(require('path').resolve(__dirname, 'web-worker-0.js'))",
+      },
+      delimiters: ['', ''],
+      preventAssignment: true,
+    }),
+    resolve({ browser: false, extensions: ['.js', '.ts'] }),
+    wasm({ targetEnv: 'node', maxFileSize: 0 }),
+    typescript({
+      compilerOptions: { outDir: 'dist/cjs', target: 'es5' },
+      exclude: ['src/worker.ts'],
+    }),
+  ],
+};

sdk/typescript/packages/mix-fetch-node/rollup-worker.config.mjs

@@ -0,0 +1,40 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import commonjs from '@rollup/plugin-commonjs';
+import modify from 'rollup-plugin-modify';
+import resolve from '@rollup/plugin-node-resolve';
+import typescript from '@rollup/plugin-typescript';
+import { wasm } from '@rollup/plugin-wasm';
+
+export default {
+  input: 'src/worker/index.ts',
+  output: {
+    dir: 'dist/cjs',
+    format: 'cjs',
+  },
+  external: ['util', 'fake-indexeddb'],
+  plugins: [
+    resolve({
+      browser: false,
+      preferBuiltins: true,
+      extensions: ['.js', '.ts'],
+    }),
+    commonjs(),
+    // TODO: One of the wasm functions calls `new WebSocket` at one point, which we aren't able to polyfill correctly yet.
+    modify({
+      find: 'const ret = new WebSocket(getStringFromWasm0(arg0, arg1));',
+      replace: 'const ws = require("ws"); const ret = new ws.WebSocket(getStringFromWasm0(arg0, arg1));',
+    }),
+    // TODO: `getObject(...).require` seems to generate a warning on Webpack but with Rollup we get a panic since it can't require.
+    // By hard coding the require here, we can workaround that.
+    // Reference: https://github.com/rust-random/getrandom/issues/224
+    modify({ find: 'getObject(arg0).require(getStringFromWasm0(arg1, arg2));', replace: 'require("crypto");' }),
+    wasm({ targetEnv: 'node', maxFileSize: 0, fileName: '[name].wasm' }),
+    typescript({
+      compilerOptions: {
+        outDir: 'dist/cjs',
+        declaration: false,
+        target: 'es5',
+      },
+    }),
+  ],
+};

sdk/typescript/packages/mix-fetch-node/scripts/build-prod-docs-collect.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+rm -rf ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch || true
+
+# run the build
+yarn docs:generate:prod
+
+# move the output outside of the yarn/npm workspaces
+mkdir -p ../../../../dist/ts/docs/tsdoc/nymproject
+mv docs ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch
+
+echo "Output can be found in:"
+realpath ../../../../dist/ts/docs/tsdoc/nymproject/mix-fetch

sdk/typescript/packages/mix-fetch-node/scripts/build-prod.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+rm -rf dist || true
+rm -rf ../../../../dist/ts/sdk/mix-fetch || true
+
+# run the build
+scripts/build.sh
+node scripts/buildPackageJson.mjs
+
+# move the output outside of the yarn/npm workspaces
+mkdir -p ../../../../dist/ts/sdk
+mv dist ../../../../dist/ts/sdk
+mv ../../../../dist/ts/sdk/dist ../../../../dist/ts/sdk/mix-fetch
+
+echo "Output can be found in:"
+realpath ../../../../dist/ts/sdk/mix-fetch

sdk/typescript/packages/mix-fetch-node/scripts/build.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+rm -rf dist || true
+
+#-------------------------------------------------------
+# WEB WORKER (mix-fetch WASM)
+#-------------------------------------------------------
+# The web worker needs to be bundled because the WASM bundle needs to be loaded synchronously and all dependencies
+# must be included in the worker script (because it is not loaded as an ES Module)
+
+# build the worker
+rollup -c rollup-worker.config.mjs
+
+# move it next to the Typescript `src/index.ts` so it can be inlined by rollup
+rm -f src/worker/*.js
+rm -f src/worker/*.wasm
+mv dist/cjs/index.js src/worker/worker.js
+
+# move WASM files out of build area
+mkdir -p dist/worker
+mv dist/cjs/*.wasm dist/worker
+
+#-------------------------------------------------------
+# COMMON JS
+#-------------------------------------------------------
+# Some old build systems cannot fully handle ESM or ES2021, so build
+# a CommonJS bundle targeting ES5
+
+# build the SDK as a CommonJS bundle
+rollup -c rollup-cjs.config.mjs
+
+# move WASM files into place
+cp dist/worker/*.wasm dist/cjs
+
+#-------------------------------------------------------
+# CLEAN UP
+#-------------------------------------------------------
+
+rm -rf dist/cjs/worker
+
+# copy README
+cp README.md dist/cjs/README.md
+
+

sdk/typescript/packages/mix-fetch-node/scripts/buildPackageJson.mjs

@@ -0,0 +1,22 @@
+import * as fs from 'fs';
+
+// parse the package.json from the SDK, so we can keep fields like the name and version
+const json = JSON.parse(fs.readFileSync('package.json').toString());
+
+// defaults (NB: these are in the output file locations)
+const browser = 'index.js';
+const main = 'index.js';
+const types = 'index.d.ts';
+
+const getPackageJson = (type, suffix) => ({
+  name: `${json.name}${suffix ? `-${suffix}` : ''}`,
+  version: json.version,
+  license: json.license,
+  author: json.author,
+  type,
+  browser,
+  main,
+  types,
+});
+
+fs.writeFileSync('dist/cjs/package.json', JSON.stringify(getPackageJson('commonjs', 'commonjs'), null, 2));

sdk/typescript/packages/mix-fetch-node/scripts/publish.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+rm -rf dist || true
+rm -rf ../../../../dist || true
+
+yarn
+yarn build
+cd ../../../../dist/sdk
+
+cd cjs
+echo "Publishing CommonJS package to NPM.."
+npm publish --access=public
+cd ..