Support loading array of custom behaviors from URL, file, or dir

Dockerfile

@@ -50,6 +50,8 @@ RUN ln -s /app/dist/main.js /usr/bin/crawl; \
     ln -s /app/dist/main.js /usr/bin/qa; \
     ln -s /app/dist/create-login-profile.js /usr/bin/create-login-profile
 
+RUN mkdir -p /app/behaviors
+
 WORKDIR /crawls
 
 # enable to test custom behaviors build (from browsertrix-behaviors)

src/crawler.ts

@@ -39,7 +39,7 @@ import {
   runWorkers,
 } from "./util/worker.js";
 import { sleep, timedRun, secondsElapsed } from "./util/timing.js";
-import { collectAllFileSources, getInfoString } from "./util/file_reader.js";
+import { collectCustomBehaviors, getInfoString } from "./util/file_reader.js";
 
 import { Browser } from "./util/browser.js";
 
@@ -510,7 +510,7 @@ export class Crawler {
     }
 
     if (this.params.customBehaviors) {
-      this.customBehaviors = this.loadCustomBehaviors(
+      this.customBehaviors = await this.loadCustomBehaviors(
         this.params.customBehaviors,
       );
     }
@@ -800,24 +800,24 @@ self.__bx_behaviors.selectMainBehavior();
     });
   }
 
-  loadCustomBehaviors(filename: string) {
+  async loadCustomBehaviors(sources: string[]) {
     let str = "";
 
-    for (const { contents } of collectAllFileSources(filename, ".js")) {
+    for (const { contents } of await collectCustomBehaviors(sources)) {
       str += `self.__bx_behaviors.load(${contents});\n`;
     }
 
     return str;
   }
 
   async checkBehaviorScripts(cdp: CDPSession) {
-    const filename = this.params.customBehaviors;
+    const sources = this.params.customBehaviors;
 
-    if (!filename) {
+    if (!sources) {
       return;
     }
 
-    for (const { path, contents } of collectAllFileSources(filename, ".js")) {
+    for (const { path, contents } of await collectCustomBehaviors(sources)) {
       await this.browser.checkScript(cdp, path, contents);
     }
   }

src/util/argParser.ts

@@ -551,8 +551,10 @@ class ArgParser {
 
         customBehaviors: {
           describe:
-            "injects a custom behavior file or set of behavior files in a directory",
-          type: "string",
+            "Custom behavior files to inject. Values can be URLs, paths to individual behavior files, or paths" +
+            "to a directory of behavior files.",
+          type: "array",
+          default: [],
         },
 
         debugAccessRedis: {

src/util/file_reader.ts

@@ -1,27 +1,78 @@
-import fs from "fs";
 import fsp from "fs/promises";
 import path from "path";
+import crypto from "crypto";
+import { fetch } from "undici";
+
+import { logger } from "./logger.js";
 
 const MAX_DEPTH = 2;
 
-export function collectAllFileSources(
-  fileOrDir: string,
-  ext?: string,
-  depth = 0,
-): { path: string; contents: string }[] {
+// Add .ts to allowed extensions when we can support it
+const ALLOWED_EXTS = [".js"];
+
+export type FileSource = {
+  path: string;
+  contents: string;
+};
+
+export type FileSources = FileSource[];
+
+export async function collectCustomBehaviors(sources: string[]) {
+  const collectedSources: FileSources = [];
+
+  for (const fileSource of sources) {
+    if (fileSource.startsWith("http")) {
+      const newSources = await collectOnlineBehavior(fileSource);
+      collectedSources.push(...newSources);
+    } else {
+      const newSources = await collectLocalPathBehaviors(fileSource);
+      collectedSources.push(...newSources);
+    }
+  }
+
+  return collectedSources;
+}
+
+async function collectOnlineBehavior(url: string) {
+  const filename = crypto.randomBytes(4).toString("hex") + ".js";
+  const behaviorFilepath = `/app/behaviors/${filename}`;
+
+  try {
+    const res = await fetch(url);
+    const fileContents = await res.text();
+    await fsp.writeFile(behaviorFilepath, fileContents);
+    logger.info(
+      "Custom behavior file downloaded",
+      { url, path: behaviorFilepath },
+      "behavior",
+    );
+    return await collectLocalPathBehaviors(behaviorFilepath);
+  } catch (e) {
+    logger.error(
+      "Error downloading custom behavior from URL",
+      { url, error: e },
+      "behavior",
+    );
+  }
+  return [];
+}
+
+async function collectLocalPathBehaviors(fileOrDir: string, depth = 0) {
   const resolvedPath = path.resolve(fileOrDir);
 
   if (depth >= MAX_DEPTH) {
-    console.warn(
-      `WARN: MAX_DEPTH of ${MAX_DEPTH} reached traversing "${resolvedPath}"`,
+    loger.warn(
+      `Max depth of ${MAX_DEPTH} reached traversing "${resolvedPath}"`,
+      {},
+      "behavior",
     );
     return [];
   }
 
-  const stat = fs.statSync(resolvedPath);
+  const stat = await fsp.stat(resolvedPath);
 
-  if (stat.isFile() && (ext === null || path.extname(resolvedPath) === ext)) {
-    const contents = fs.readFileSync(resolvedPath);
+  if (stat.isFile() && ALLOWED_EXTS.includes(path.extname(resolvedPath))) {
+    const contents = await fsp.readFile(resolvedPath);
     return [
       {
         path: resolvedPath,
@@ -31,19 +82,18 @@ export function collectAllFileSources(
   }
 
   if (stat.isDirectory()) {
-    const files = fs.readdirSync(resolvedPath);
-    return files.reduce(
-      (acc: { path: string; contents: string }[], next: string) => {
-        const nextPath = path.join(fileOrDir, next);
-        return [...acc, ...collectAllFileSources(nextPath, ext, depth + 1)];
-      },
-      [],
-    );
+    const files = await fsp.readdir(resolvedPath);
+    return files.reduce((acc: FileSources, next: string) => {
+      const nextPath = path.join(fileOrDir, next);
+      return [...acc, ...collectLocalPathBehaviors(nextPath, depth + 1)];
+    }, []);
   }
 
   if (depth === 0) {
-    console.warn(
+    logger.warn(
       `WARN: The provided path "${resolvedPath}" is not a .js file or directory.`,
+      {},
+      "behavior",
     );
   }
 

tests/custom-behavior.test.js

@@ -1,6 +1,6 @@
 import child_process from "child_process";
 
-test("test custom behaviors", async () => {
+test("test custom behaviors from local filepath", async () => {
   const res = child_process.execSync(
     "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ webrecorder/browsertrix-crawler crawl --url https://example.com/ --url https://example.org/ --url https://webrecorder.net/ --customBehaviors /custom-behaviors/ --scopeType page",
   );
@@ -35,6 +35,30 @@ test("test custom behaviors", async () => {
   ).toBe(true);
 });
 
+test("test custom behavior from URL", async () => {
+  const res = child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/ --customBehaviors https://raw.githubusercontent.com/webrecorder/browsertrix-behaviors/main/dist/behaviors.js --scopeType page --behaviors \"\"");
+
+  const log = res.toString();
+
+  expect(log.indexOf("Custom behavior file downloaded") > 0).toBe(true);
+});
+
+test("test mixed custom behavior sources", async () => {
+  const res = child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/ --customBehaviors https://raw.githubusercontent.com/webrecorder/browsertrix-behaviors/main/dist/behaviors.js --customBehaviors /custom-behaviors/custom-2.js --scopeType page --behaviors \"\"");
+
+  const log = res.toString();
+
+  // test custom behavior from url ran
+  expect(log.indexOf("Custom behavior file downloaded") > 0).toBe(true);
+
+  // test custom behavior from local file ran
+  expect(
+    log.indexOf(
+      '{"state":{},"msg":"test-stat-2","page":"https://webrecorder.net/","workerid":0}}',
+    ) > 0,
+  ).toBe(true);
+});
+
 test("test invalid behavior exit", async () => {
   let status = 0;