diff --git a/crawler.js b/crawler.js index 0b9d8af84..03c0fa451 100644 --- a/crawler.js +++ b/crawler.js @@ -163,7 +163,7 @@ export class Crawler { } catch (e) { //logger.fatal("Unable to connect to state store Redis: " + redisUrl); logger.warn(`Waiting for redis at ${redisUrl}`, {}, "state"); - await sleep(3); + await sleep(1); } } @@ -304,12 +304,16 @@ export class Crawler { async run() { await this.bootstrap(); - let status; + let status = "done"; let exitCode = 0; try { await this.crawl(); - status = (!this.interrupted ? "done" : "interrupted"); + const finished = await this.crawlState.isFinished(); + if (this.interrupted && !finished) { + status = "interrupted"; + exitCode = 11; + } } catch(e) { logger.error("Crawl failed", e); exitCode = 9; diff --git a/util/state.js b/util/state.js index 5c33871ca..0cfe45471 100644 --- a/util/state.js +++ b/util/state.js @@ -207,7 +207,15 @@ return 0; } async isCrawlStopped() { - return await this.redis.get(`${this.key}:stopping`) === "1"; + if (await this.redis.get(`${this.key}:stopping`) === "1") { + return true; + } + + if (await this.redis.hget(`${this.key}:stopone`, this.uid) === "1") { + return true; + } + + return false; } // note: not currently called in crawler, but could be