diff --git a/crawler.js b/crawler.js index c82a8ca0c..c3b0acf1f 100644 --- a/crawler.js +++ b/crawler.js @@ -25,7 +25,7 @@ import { collectAllFileSources } from "./util/file_reader.js"; import { Browser } from "./util/browser.js"; -import { BEHAVIOR_LOG_FUNC, HTML_TYPES, DEFAULT_SELECTORS } from "./util/constants.js"; +import { ADD_LINK_FUNC, BEHAVIOR_LOG_FUNC, HTML_TYPES, DEFAULT_SELECTORS } from "./util/constants.js"; import { AdBlockRules, BlockRules } from "./util/blockrules.js"; import { OriginOverride } from "./util/originoverride.js"; @@ -381,7 +381,7 @@ export class Crawler { return seed.isIncluded(url, depth, extraHops, logDetails); } - async setupPage({page, cdp, workerid}) { + async setupPage({page, cdp, workerid, callbacks}) { await this.browser.setupPage({page, cdp}); if ((this.adBlockRules && this.params.blockAds) || @@ -419,6 +419,8 @@ export class Crawler { await this.screencaster.screencastPage(page, cdp, workerid); } + await page.exposeFunction(ADD_LINK_FUNC, (url) => callbacks.addLink && callbacks.addLink(url)); + if (this.params.behaviorOpts) { await page.exposeFunction(BEHAVIOR_LOG_FUNC, (logdata) => this._behaviorLog(logdata, page.url(), workerid)); await this.browser.addInitScript(page, behaviors); @@ -459,7 +461,8 @@ self.__bx_behaviors.selectMainBehavior(); async crawlPage(opts) { await this.writeStats(); - const {page, cdp, data, workerid} = opts; + const {page, cdp, data, workerid, callbacks} = opts; + data.callbacks = callbacks; const {url} = data; @@ -1056,7 +1059,7 @@ self.__bx_behaviors.selectMainBehavior(); } async loadPage(page, data, selectorOptsList = DEFAULT_SELECTORS) { - const {url, seedId, depth, extraHops = 0} = data; + const {url, seedId, depth} = data; const logDetails = data.logDetails; @@ -1186,12 +1189,9 @@ self.__bx_behaviors.selectMainBehavior(); return; } - logger.debug("Extracting links"); + logger.debug("Extracting links", logDetails); - for (const opts of selectorOptsList) { - const links = await this.extractLinks(page, data.filteredFrames, opts, logDetails); - await this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails); - } + await this.extractLinks(page, data, selectorOptsList, logDetails); } async netIdle(page, details) { @@ -1210,52 +1210,66 @@ self.__bx_behaviors.selectMainBehavior(); } } - async extractLinks(page, frames, {selector = "a[href]", extract = "href", isAttribute = false} = {}, logDetails) { - const results = []; + async extractLinks(page, data, selectors = DEFAULT_SELECTORS, logDetails) { + const {seedId, depth, extraHops = 0, filteredFrames, callbacks} = data; - const loadProp = (options) => { - const { selector, extract } = options; - return [...document.querySelectorAll(selector)].map(elem => elem[extract]); - }; + let links = []; + const promiseList = []; - const loadAttr = (options) => { - const { selector, extract } = options; - return [...document.querySelectorAll(selector)].map(elem => elem.getAttribute(extract)); + callbacks.addLink = (url) => { + links.push(url); + if (links.length == 500) { + promiseList.push(this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails)); + links = []; + } }; - const loadFunc = isAttribute ? loadAttr : loadProp; + const loadLinks = (options) => { + const { selector, extract, isAttribute, addLinkFunc } = options; + const urls = new Set(); + + const getAttr = elem => urls.add(elem.getAttribute(extract)); + const getProp = elem => urls.add(elem[extract]); + + const getter = isAttribute ? getAttr : getProp; - frames = frames || page.frames(); + document.querySelectorAll(selector).forEach(getter); + + const func = window[addLinkFunc]; + urls.forEach(url => func.call(this, url)); + + return true; + }; + + const frames = filteredFrames || page.frames(); try { - const linkResults = await Promise.allSettled( - frames.map(frame => timedRun( - frame.evaluate(loadFunc, {selector, extract}), - PAGE_OP_TIMEOUT_SECS, - "Link extraction timed out", - logDetails, - )) - ); + for (const {selector = "a[href]", extract = "href", isAttribute = false} of selectors) { + const promiseResults = await Promise.allSettled( + frames.map(frame => timedRun( + frame.evaluate(loadLinks, {selector, extract, isAttribute, addLinkFunc: ADD_LINK_FUNC}), + PAGE_OP_TIMEOUT_SECS, + "Link extraction timed out", + logDetails, + )) + ); - if (linkResults) { - let i = 0; - for (const linkResult of linkResults) { - if (!linkResult) { - logger.warn("Link Extraction timed out in frame", {frameUrl: frames[i].url, ...logDetails}); - continue; - } - if (!linkResult.value) continue; - for (const link of linkResult.value) { - results.push(link); + for (let i = 0; i < promiseResults.length; i++) { + const {status, reason} = promiseResults[i]; + if (status === "rejected") { + logger.warn("Link Extraction failed in frame", {reason, frameUrl: frames[i].url, ...logDetails}); } - i++; } } - } catch (e) { logger.warn("Link Extraction failed", e); } - return results; + + if (links.length) { + promiseList.push(this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails)); + } + + await Promise.allSettled(promiseList); } async queueInScopeUrls(seedId, urls, depth, extraHops = 0, logDetails = {}) { @@ -1304,22 +1318,17 @@ self.__bx_behaviors.selectMainBehavior(); } async queueUrl(seedId, url, depth, extraHops = 0) { - logger.debug(`Queuing url ${url}`); if (this.limitHit) { return false; } - if (this.pageLimit > 0 && (await this.crawlState.numSeen() >= this.pageLimit)) { + if (!await this.crawlState.addToQueue({url, seedId, depth, extraHops}, this.pageLimit)) { this.limitHit = true; return false; + } else { + logger.debug(`Queued url ${url}`); } - if (await this.crawlState.has(url)) { - return false; - } - - //await this.crawlState.add(url); - await this.crawlState.addToQueue({url, seedId, depth, extraHops}); return true; } diff --git a/util/browser.js b/util/browser.js index 806c0f174..4d5f6fcc0 100644 --- a/util/browser.js +++ b/util/browser.js @@ -51,6 +51,7 @@ export class BaseBrowser handleSIGHUP: signals, handleSIGINT: signals, handleSIGTERM: signals, + protocolTimeout: 0, defaultViewport, waitForInitialPage: false, diff --git a/util/constants.js b/util/constants.js index c916c3394..6cef3078b 100644 --- a/util/constants.js +++ b/util/constants.js @@ -2,6 +2,7 @@ export const HTML_TYPES = ["text/html", "application/xhtml", "application/xhtml+xml"]; export const WAIT_UNTIL_OPTS = ["load", "domcontentloaded", "networkidle0", "networkidle2"]; export const BEHAVIOR_LOG_FUNC = "__bx_log"; +export const ADD_LINK_FUNC = "__bx_addLink"; export const MAX_DEPTH = 1000000; export const DEFAULT_SELECTORS = [{ diff --git a/util/state.js b/util/state.js index 0cfe45471..9c9b1989b 100644 --- a/util/state.js +++ b/util/state.js @@ -67,9 +67,17 @@ export class RedisCrawlState redis.defineCommand("addqueue", { numberOfKeys: 3, lua: ` -redis.call('sadd', KEYS[3], ARGV[1]); +local size = redis.call('scard', KEYS[3]); +local limit = tonumber(ARGV[4]); +if limit > 0 and size >= limit then + return 0; +end +if redis.call('sadd', KEYS[3], ARGV[1]) == 0 then + return 0; +end redis.call('zadd', KEYS[2], ARGV[2], ARGV[3]); redis.call('hdel', KEYS[1], ARGV[1]); +return 1; ` }); @@ -233,14 +241,14 @@ return 0; return (res >= 3); } - async addToQueue({url, seedId, depth = 0, extraHops = 0} = {}) { + async addToQueue({url, seedId, depth = 0, extraHops = 0} = {}, limit = 0) { const added = this._timestamp(); const data = {added, url, seedId, depth}; if (extraHops) { data.extraHops = extraHops; } - await this.redis.addqueue(this.pkey, this.qkey, this.skey, url, this._getScore(data), JSON.stringify(data)); + return await this.redis.addqueue(this.pkey, this.qkey, this.skey, url, this._getScore(data), JSON.stringify(data), limit); } async nextFromQueue() { diff --git a/util/worker.js b/util/worker.js index 6117956ec..d03c332f2 100644 --- a/util/worker.js +++ b/util/worker.js @@ -48,6 +48,7 @@ export class PageWorker this.reuseCount = 0; this.page = null; this.cdp = null; + this.callbacks = null; this.opts = null; @@ -115,7 +116,8 @@ export class PageWorker this.page = page; this.cdp = cdp; - this.opts = {page: this.page, cdp: this.cdp, workerid}; + this.callbacks = {}; + this.opts = {page: this.page, cdp: this.cdp, workerid, callbacks: this.callbacks}; // updated per page crawl this.crashed = false; @@ -196,6 +198,8 @@ export class PageWorker async runLoop() { const crawlState = this.crawler.crawlState; + let loggedWaiting = false; + while (await this.crawler.isCrawlRunning()) { const data = await crawlState.nextFromQueue(); @@ -207,6 +211,8 @@ export class PageWorker // run timed crawl of page await this.timedCrawlPage({...opts, data}); + loggedWaiting = false; + } else { // indicate that the worker has no more work (mostly for screencasting, status, etc...) // depending on other works, will either get more work or crawl will end @@ -217,7 +223,10 @@ export class PageWorker // if pending, sleep and check again if (pending) { - logger.debug("No crawl tasks, but pending tasks remain, waiting", {pending, workerid: this.id}, "worker"); + if (!loggedWaiting) { + logger.debug("No crawl tasks, but pending tasks remain, waiting", {pending, workerid: this.id}, "worker"); + loggedWaiting = true; + } await sleep(0.5); } else { // if no pending and queue size is still empty, we're done! diff --git a/yarn.lock b/yarn.lock index 7e79d4c6f..ccdf6872f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -620,16 +620,16 @@ resolved "https://registry.yarnpkg.com/@novnc/novnc/-/novnc-1.4.0.tgz#68adae81a741624142b518323441e852c1f34281" integrity sha512-kW6ALMc5BuH08e/ond/I1naYcfjc19JYMN1EdtmgjjjzPGCjW8fMtVM3MwM6q7YLRjPlQ3orEvoKMgSS7RkEVQ== -"@puppeteer/browsers@1.4.3": - version "1.4.3" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-1.4.3.tgz#39bfd8bf999d707ed2914b036fa2febac2960985" - integrity sha512-8Jfkpb8qhPQhMsNBmIY8b6+ic2kvcmHZlyvifmcNKBC5jNZf3MAKq3gryKfmrjFAYFl3naPjiKljPUq5wuolfQ== +"@puppeteer/browsers@1.7.1": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-1.7.1.tgz#04f1e3aec4b87f50a7acc8f64be2149bda014f0a" + integrity sha512-nIb8SOBgDEMFY2iS2MdnUZOg2ikcYchRrBoF+wtdjieRFKR2uGRipHY/oFLo+2N6anDualyClPzGywTHRGrLfw== dependencies: debug "4.3.4" extract-zip "2.0.1" progress "2.0.3" - proxy-agent "6.2.1" - tar-fs "3.0.3" + proxy-agent "6.3.1" + tar-fs "3.0.4" unbzip2-stream "1.4.3" yargs "17.7.1" @@ -664,6 +664,11 @@ dependencies: defer-to-connect "^2.0.0" +"@tootallnate/quickjs-emscripten@^0.23.0": + version "0.23.0" + resolved "https://registry.yarnpkg.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz#db4ecfd499a9765ab24002c3b696d02e6d32a12c" + integrity sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA== + "@types/babel__core@^7.1.14": version "7.1.19" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.19.tgz#7b497495b7d1b4812bdb9d02804d0576f43ee460" @@ -796,22 +801,12 @@ acorn-jsx@^5.3.1: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== -acorn-walk@^8.2.0: - version "8.2.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" - integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== - acorn@^7.4.0: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.7.0: - version "8.8.2" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" - integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== - -agent-base@^7.0.1, agent-base@^7.0.2, agent-base@^7.1.0: +agent-base@^7.0.2, agent-base@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.0.tgz#536802b76bc0b34aa50195eb2442276d613e3434" integrity sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg== @@ -930,7 +925,7 @@ asn1.js@^5.2.0: minimalistic-assert "^1.0.0" safer-buffer "^2.1.0" -ast-types@^0.13.2: +ast-types@^0.13.4: version "0.13.4" resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.13.4.tgz#ee0d77b343263965ecc3fb62da16e7222b2b6782" integrity sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w== @@ -1260,12 +1255,12 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== -chromium-bidi@0.4.16: - version "0.4.16" - resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.4.16.tgz#8a67bfdf6bb8804efc22765a82859d20724b46ab" - integrity sha512-7ZbXdWERxRxSwo3txsBjjmc/NLxqb1Bk30mRb0BMS4YIaiV6zvKZqL/UAH+DdqcDYayDWk2n/y8klkBDODrPvA== +chromium-bidi@0.4.26: + version "0.4.26" + resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.4.26.tgz#4271d2c2507ad86ad08cbccf4c7e3b8a1b1e8112" + integrity sha512-lukBGfogAI4T0y3acc86RaacqgKQve47/8pV2c+Hr1PjcICj2K4OkL3qfX3qrqxxnd4ddurFC0WBA3VCQqYeUQ== dependencies: - mitt "3.0.0" + mitt "3.0.1" ci-info@^3.2.0: version "3.5.0" @@ -1408,12 +1403,12 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" -cross-fetch@3.1.6: - version "3.1.6" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.6.tgz#bae05aa31a4da760969756318feeee6e70f15d6c" - integrity sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g== +cross-fetch@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" + integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== dependencies: - node-fetch "^2.6.11" + node-fetch "^2.6.12" cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" @@ -1492,11 +1487,6 @@ deep-is@^0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= -deep-is@~0.1.3: - version "0.1.4" - resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" - integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== - deepmerge@^4.2.2: version "4.2.2" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" @@ -1514,15 +1504,14 @@ define-properties@^1.1.3: dependencies: object-keys "^1.0.12" -degenerator@^4.0.1: - version "4.0.2" - resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-4.0.2.tgz#55b7fb41239ee0ea7644fa3f2aba84e0adfaa40c" - integrity sha512-HKwIFvZROUMfH3qI3gBpD61BYh7q3c3GXD5UGZzoVNJwVSYgZKvYl1fRMXc9ozoTxl/VZxKJ5v/bA+19tywFiw== +degenerator@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-5.0.1.tgz#9403bf297c6dad9a1ece409b37db27954f91f2f5" + integrity sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ== dependencies: - ast-types "^0.13.2" - escodegen "^1.8.1" - esprima "^4.0.0" - vm2 "^3.9.17" + ast-types "^0.13.4" + escodegen "^2.1.0" + esprima "^4.0.1" denque@^1.1.0: version "1.5.0" @@ -1547,10 +1536,10 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -devtools-protocol@0.0.1135028: - version "0.0.1135028" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1135028.tgz#b2c667c301cb6da9ba3ed0989fe1fb88b660ee0a" - integrity sha512-jEcNGrh6lOXNRJvZb9RjeevtZGrgugPKSMJZxfyxWQnhlKawMPhMtk/dfC+Z/6xNXExlzTKlY5LzIAK/fRpQIw== +devtools-protocol@0.0.1159816: + version "0.0.1159816" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1159816.tgz#b5848e8597de01e4738589e7553674c7312c8d2a" + integrity sha512-2cZlHxC5IlgkIWe2pSDmCrDiTzbSJWywjbDDnupOImEBcG31CQgBLV8wWE+5t+C4rimcjHsbzy7CBzf9oFjboA== diff-sequences@^29.2.0: version "29.2.0" @@ -1706,15 +1695,14 @@ escape-string-regexp@^2.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== -escodegen@^1.8.1: - version "1.14.3" - resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" - integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw== +escodegen@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.1.0.tgz#ba93bbb7a43986d29d6041f99f5262da773e2e17" + integrity sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w== dependencies: esprima "^4.0.1" - estraverse "^4.2.0" + estraverse "^5.2.0" esutils "^2.0.2" - optionator "^0.8.1" optionalDependencies: source-map "~0.6.1" @@ -1832,7 +1820,7 @@ esrecurse@^4.3.0: dependencies: estraverse "^5.2.0" -estraverse@^4.1.1, estraverse@^4.2.0: +estraverse@^4.1.1: version "4.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== @@ -1917,7 +1905,7 @@ fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: +fast-levenshtein@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= @@ -2242,10 +2230,10 @@ http2-wrapper@^1.0.0-beta.5.2: quick-lru "^5.1.1" resolve-alpn "^1.0.0" -https-proxy-agent@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.0.tgz#75cb70d04811685667183b31ab158d006750418a" - integrity sha512-0euwPCRyAPSgGdzD1IVN9nJYHtBhJwb6XPfbpQcYbPCwrBidX6GzxmchnaF4sfF/jPb74Ojx5g4yTg3sixlyPw== +https-proxy-agent@^7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" + integrity sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA== dependencies: agent-base "^7.0.2" debug "4" @@ -2334,7 +2322,7 @@ ioredis@^4.27.1: redis-parser "^3.0.0" standard-as-callback "^2.1.0" -ip@^1.1.5: +ip@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.8.tgz#ae05948f6b075435ed3307acce04629da8cdbf48" integrity sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg== @@ -2442,6 +2430,11 @@ is-glob@^4.0.0, is-glob@^4.0.1: dependencies: is-extglob "^2.1.1" +is-gzip@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-gzip/-/is-gzip-2.0.0.tgz#f4fed2bbd9f96bf2cb39e19262797fdb15aad933" + integrity sha512-jtO4Njg6q58zDo/Pu4027beSZ0VdsZlt8/5Moco6yAg+DIxb5BK/xUYqYG2+MD4+piKldXJNHxRkhEYI2fvrxA== + is-negative-zero@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" @@ -3033,14 +3026,6 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -levn@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA== - dependencies: - prelude-ls "~1.1.2" - type-check "~0.3.2" - lines-and-columns@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" @@ -3231,10 +3216,10 @@ minio@7.0.26: xml "^1.0.0" xml2js "^0.4.15" -mitt@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.0.tgz#69ef9bd5c80ff6f57473e8d89326d01c414be0bd" - integrity sha512-7dX2/10ITVyqh4aOSVI9gdape+t9l2/8QxHrFmUXu4EEUpdlxl6RudZUPZoc+zuY2hk1j7XxVroIVIan/pD/SQ== +mitt@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.1.tgz#ea36cf0cc30403601ae074c8f77b7092cdab36d1" + integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw== mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" @@ -3280,10 +3265,10 @@ node-addon-api@^6.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== -node-fetch@^2.6.11: - version "2.6.11" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25" - integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w== +node-fetch@^2.6.12: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== dependencies: whatwg-url "^5.0.0" @@ -3388,18 +3373,6 @@ onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" -optionator@^0.8.1: - version "0.8.3" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" - integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== - dependencies: - deep-is "~0.1.3" - fast-levenshtein "~2.0.6" - levn "~0.3.0" - prelude-ls "~1.1.2" - type-check "~0.3.2" - word-wrap "~1.2.3" - optionator@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" @@ -3448,26 +3421,27 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== -pac-proxy-agent@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-6.0.3.tgz#61042187093b67aa7dd05b41e4ec7c241a27c428" - integrity sha512-5Hr1KgPDoc21Vn3rsXBirwwDnF/iac1jN/zkpsOYruyT+ZgsUhUOgVwq3v9+ukjZd/yGm/0nzO1fDfl7rkGoHQ== +pac-proxy-agent@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz#6b9ddc002ec3ff0ba5fdf4a8a21d363bcc612d75" + integrity sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A== dependencies: + "@tootallnate/quickjs-emscripten" "^0.23.0" agent-base "^7.0.2" debug "^4.3.4" get-uri "^6.0.1" http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" - pac-resolver "^6.0.1" - socks-proxy-agent "^8.0.1" + https-proxy-agent "^7.0.2" + pac-resolver "^7.0.0" + socks-proxy-agent "^8.0.2" -pac-resolver@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-6.0.1.tgz#319c182d3db4e6782e79519cb4dd1dda46579292" - integrity sha512-dg497MhVT7jZegPRuOScQ/z0aV/5WR0gTdRu1md+Irs9J9o+ls5jIuxjo1WfaTG+eQQkxyn5HMGvWK+w7EIBkQ== +pac-resolver@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.0.tgz#79376f1ca26baf245b96b34c339d79bff25e900c" + integrity sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg== dependencies: - degenerator "^4.0.1" - ip "^1.1.5" + degenerator "^5.0.0" + ip "^1.1.8" netmask "^2.0.2" pako@^1.0.11: @@ -3594,11 +3568,6 @@ prelude-ls@^1.2.1: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== -prelude-ls@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== - pretty-format@^29.2.1: version "29.2.1" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.2.1.tgz#86e7748fe8bbc96a6a4e04fa99172630907a9611" @@ -3630,19 +3599,19 @@ prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.8.1" -proxy-agent@6.2.1: - version "6.2.1" - resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.2.1.tgz#062df6609a4012fd1c108974865599b61e77abde" - integrity sha512-OIbBKlRAT+ycCm6wAYIzMwPejzRtjy8F3QiDX0eKOA3e4pe3U9F/IvzcHP42bmgQxVv97juG+J8/gx+JIeCX/Q== +proxy-agent@6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.1.tgz#40e7b230552cf44fd23ffaf7c59024b692612687" + integrity sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ== dependencies: agent-base "^7.0.2" debug "^4.3.4" http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" + https-proxy-agent "^7.0.2" lru-cache "^7.14.1" - pac-proxy-agent "^6.0.3" + pac-proxy-agent "^7.0.1" proxy-from-env "^1.1.0" - socks-proxy-agent "^8.0.1" + socks-proxy-agent "^8.0.2" proxy-from-env@^1.1.0: version "1.1.0" @@ -3674,17 +3643,17 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -puppeteer-core@^20.7.4: - version "20.7.4" - resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-20.7.4.tgz#4ea245aa8746873584f620222fc31a1781394b32" - integrity sha512-7YZ1LmTo+5yM9uBNFTMJpE+lJjcIoNjKVarsYIk7o5WhgQNI9o5XgiQK5f71y1vWwr7sT/eGG75HXAehjnTBTg== +puppeteer-core@^21.2.1: + version "21.2.1" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-21.2.1.tgz#2b8a3d4bafa15707c74d67e24a5a10a5577e78b4" + integrity sha512-+I8EjpWFeeFKScpQiTEnC4jGve2Wr4eA9qUMoa8S317DJPm9h7wzrT4YednZK2TQZMyPtPQ2Disb/Tg02+4Naw== dependencies: - "@puppeteer/browsers" "1.4.3" - chromium-bidi "0.4.16" - cross-fetch "3.1.6" + "@puppeteer/browsers" "1.7.1" + chromium-bidi "0.4.26" + cross-fetch "4.0.0" debug "4.3.4" - devtools-protocol "0.0.1135028" - ws "8.13.0" + devtools-protocol "0.0.1159816" + ws "8.14.1" querystring@0.2.0: version "0.2.0" @@ -3990,12 +3959,14 @@ sisteransi@^1.0.5: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== -sitemapper@^3.1.2: - version "3.1.12" - resolved "https://registry.yarnpkg.com/sitemapper/-/sitemapper-3.1.12.tgz#58f6cb22112da4d73498c3d14c926c3a1c0d74ee" - integrity sha512-0BOXAhIfjQll1rrUkkFkpAhYy7MTs887H7Zpc4eAxLPPJJRFXNDdrryTadFWubGMn62bqGr3KdKBKhVdtt/HWg== +sitemapper@^3.2.6: + version "3.2.6" + resolved "https://registry.yarnpkg.com/sitemapper/-/sitemapper-3.2.6.tgz#892ebdade9a1b0839bd3dee3b67f3d57b10b3a89" + integrity sha512-AZbim4lmKgchUj6yyJ9ru0eLJ4/S6QAqy5QEbpCpvBbBnXxTERLMC6rzgKy1gHM19YUEtYJFTC2t8lxDWO0wkQ== dependencies: got "^11.8.0" + is-gzip "2.0.0" + p-limit "^3.1.0" xml2js "^0.4.23" slash@^3.0.0: @@ -4017,12 +3988,12 @@ smart-buffer@^4.2.0: resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== -socks-proxy-agent@^8.0.1: - version "8.0.1" - resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.1.tgz#ffc5859a66dac89b0c4dab90253b96705f3e7120" - integrity sha512-59EjPbbgg8U3x62hhKOFVAmySQUcfRQ4C7Q/D5sEHnZTQRrQlNKINks44DMR1gwXp0p4LaVIeccX2KHTTcHVqQ== +socks-proxy-agent@^8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad" + integrity sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g== dependencies: - agent-base "^7.0.1" + agent-base "^7.0.2" debug "^4.3.4" socks "^2.7.1" @@ -4206,14 +4177,14 @@ table@^6.0.4: string-width "^4.2.0" strip-ansi "^6.0.0" -tar-fs@3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.3.tgz#6593e7df92f337e74d755be183a192213d923050" - integrity sha512-ZK36riGYnFI6LujIBfBRoDfeaaWUkStIFKwtPjnDWCKnsDE9kuQthG09aQjLjpzoRtVElEMZ/AIAURNb7N9mkA== +tar-fs@3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" + integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== dependencies: mkdirp-classic "^0.5.2" pump "^3.0.0" - tar-stream "^3.1.0" + tar-stream "^3.1.5" tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" @@ -4236,7 +4207,7 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" -tar-stream@^3.1.0: +tar-stream@^3.1.5: version "3.1.6" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.6.tgz#6520607b55a06f4a2e2e04db360ba7d338cc5bab" integrity sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg== @@ -4313,13 +4284,6 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" -type-check@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg== - dependencies: - prelude-ls "~1.1.2" - type-detect@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" @@ -4419,14 +4383,6 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^1.6.0" -vm2@^3.9.17: - version "3.9.19" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.19.tgz#be1e1d7a106122c6c492b4d51c2e8b93d3ed6a4a" - integrity sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg== - dependencies: - acorn "^8.7.0" - acorn-walk "^8.2.0" - walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" @@ -4501,7 +4457,7 @@ which@^2.0.1: dependencies: isexe "^2.0.0" -word-wrap@^1.2.3, word-wrap@~1.2.3: +word-wrap@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== @@ -4537,10 +4493,10 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@8.13.0: - version "8.13.0" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.13.0.tgz#9a9fb92f93cf41512a0735c8f4dd09b8a1211cd0" - integrity sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA== +ws@8.14.1: + version "8.14.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.1.tgz#4b9586b4f70f9e6534c7bb1d3dc0baa8b8cf01e0" + integrity sha512-4OOseMUq8AzRBI/7SLMUwO+FEDnguetSk7KMb1sHwvF2w2Wv5Hoj0nlifx8vtGsftE/jWHojPy8sMMzYLJ2G/A== ws@^7.4.4: version "7.5.9"