diff --git a/crawler.js b/crawler.js index 03c0fa451..da3f7ca49 100644 --- a/crawler.js +++ b/crawler.js @@ -323,7 +323,7 @@ export class Crawler { } } finally { - logger.info(`Crawl status: ${status}`); + logger.info(`Final crawl status: ${status}`); if (this.crawlState) { await this.crawlState.setStatus(status); @@ -830,7 +830,7 @@ self.__bx_behaviors.selectMainBehavior(); } } - await this.closeLog(); + logger.info("Crawling done"); if (this.params.generateWACZ && (!this.interrupted || this.finalExit || this.uploadAndDeleteLocal)) { const uploaded = await this.generateWACZ(); @@ -855,16 +855,6 @@ self.__bx_behaviors.selectMainBehavior(); } } - async closeLog() { - // close file-based log - logger.setExternalLogStream(null); - try { - await new Promise(resolve => this.logFH.close(() => resolve())); - } catch (e) { - // ignore - } - } - async generateWACZ() { logger.info("Generating WACZ"); await this.crawlState.setStatus("generate-wacz"); @@ -890,6 +880,8 @@ self.__bx_behaviors.selectMainBehavior(); logger.fatal("No WARC Files, assuming crawl failed"); } + logger.debug("End of log file, storing logs in WACZ"); + // Build the argument list to pass to the wacz create command const waczFilename = this.params.collection.concat(".wacz"); const waczPath = path.join(this.collDir, waczFilename);