Set new logic for invalid seeds (#395)

Allow for some seeds to be invalid unless failOnFailedSeed is set Fail crawl if not valid seeds are provided Co-authored-by: Ilya Kreymer <[email protected]>
webrecorder · Sep 29, 2023 · 7e03dc0 · 7e03dc0
1 parent 18dce95
commit 7e03dc0
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 2 deletions.
diff --git a/tests/seeds.test.js b/tests/seeds.test.js
@@ -0,0 +1,38 @@
+import util from "util";
+import {exec as execCallback } from "child_process";
+
+const exec = util.promisify(execCallback);
+
+
+test("ensure one invalid seed doesn't end crawl if failOnFailedSeed is not set", async () => {
+  let passed = true;
+  try {
+    await exec("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection invalidseed");
+  } catch (error) {
+    console.log(error);
+    passed = false;
+  }
+  expect(passed).toBe(true);
+});
+
+test("ensure one invalid seed fails crawl if failOnFailedSeed is set", async () => {
+  let passed = true;
+  try {
+    await exec("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --failOnFailedSeed --collection failseed");
+  }
+  catch (error) {
+    passed = false;
+  }
+  expect(passed).toBe(false);
+});
+
+test("ensure crawl fails if no valid seeds are passed", async () => {
+  let passed = true;
+  try {
+    await exec("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection allinvalidseeds");
+  }
+  catch (error) {
+    passed = false;
+  }
+  expect(passed).toBe(false);
+});
diff --git a/util/argParser.js b/util/argParser.js
@@ -537,7 +537,18 @@ class ArgParser {
       if (typeof(seed) === "string") {
         seed = {url: seed};
       }
-      argv.scopedSeeds.push(new ScopedSeed({...scopeOpts, ...seed}));
+
+      try {
+        argv.scopedSeeds.push(new ScopedSeed({...scopeOpts, ...seed}));
+      } catch (e) {
+        if (argv.failOnFailedSeed) {
+          logger.fatal(`Invalid Seed "${seed.url}" specified, aborting crawl.`);
+        }
+      }
+    }
+
+    if (!argv.scopedSeeds.length) {
+      logger.fatal("No valid seeds specified, aborting crawl.");
     }
 
     // Resolve statsFilename

diff --git a/util/seeds.js b/util/seeds.js
@@ -7,7 +7,7 @@ export class ScopedSeed
   constructor({url, scopeType, include, exclude = [], allowHash = false, depth = -1, sitemap = false, extraHops = 0} = {}) {
     const parsedUrl = this.parseUrl(url);
     if (!parsedUrl) {
-      logger.fatal(`Invalid Seed "${url}" specified, aborting crawl.`);
+      throw new Error("Invalid URL");
     }
     this.url = parsedUrl.href;
     this.include = this.parseRx(include);