diff --git a/bin/cli.js b/bin/cli.js index 1a8b88b6..23cedea3 100644 --- a/bin/cli.js +++ b/bin/cli.js @@ -27,6 +27,10 @@ const startScraper = async argv => { if (argv.historypath) { argv.historyPath = argv.historypath; } + if (argv.throttleinterval || argv.throttlelimit) { + argv.throttleLimit = argv.throttlelimit; + argv.throttleInterval = argv.throttleinterval; + } if (argv.file) { argv.input = argv.file; } @@ -205,6 +209,12 @@ yargs default: process.env.SCRAPING_FROM_DOCKER ? '' : tmpdir(), describe: 'Set custom path where history file/files will be stored', }, + throttlelimit: { + describe: 'Set custom maximum number of calls to TikTok within an interval.', + }, + throttleinterval: { + describe: 'Set custom timespan for throttle-interval in milliseconds', + }, remove: { alias: ['r'], default: '', diff --git a/package.json b/package.json index a70884e5..bae82802 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "jsdom": "^16.5.3", "json2csv": "4.5.1", "ora": "^4.0.2", + "p-throttle": "^4.1.1", "progress": "^2.0.3", "request": "^2.88.0", "request-promise": "^4.2.4", diff --git a/src/core/TikTok.ts b/src/core/TikTok.ts index 6b67ccca..766f3c48 100644 --- a/src/core/TikTok.ts +++ b/src/core/TikTok.ts @@ -13,6 +13,7 @@ import { EventEmitter } from 'events'; import { SocksProxyAgent } from 'socks-proxy-agent'; import { forEachLimit } from 'async'; import { URLSearchParams } from 'url'; +import pThrottle from 'p-throttle'; import CONST from '../constant'; import { sign, makeid } from '../helpers'; @@ -125,6 +126,8 @@ export class TikTokScraper extends EventEmitter { private store: string[]; + private throttle: ReturnType | false; + public cookieJar: CookieJar; constructor({ @@ -157,6 +160,8 @@ export class TikTokScraper extends EventEmitter { headers, verifyFp = '', sessionList = [], + throttleLimit, + throttleInterval, }: TikTokConstructor) { super(); this.userIdStore = ''; @@ -221,6 +226,16 @@ export class TikTokScraper extends EventEmitter { bad: 0, }; this.store = []; + this.throttle = + !!(throttleLimit && throttleInterval) && + pThrottle({ + limit: throttleLimit, + interval: throttleInterval, + }); + + if (this.throttle) { + this.request = this.throttle(this.request); + } } /** diff --git a/src/types/TikTok.ts b/src/types/TikTok.ts index 6b315cbc..8f19f05f 100644 --- a/src/types/TikTok.ts +++ b/src/types/TikTok.ts @@ -43,6 +43,8 @@ export interface Options { remove?: string; fileName?: string; historyPath?: string; + throttleLimit?: number; + throttleInterval?: number; timeout?: number; hdVideo?: boolean; randomUa?: boolean; @@ -83,6 +85,8 @@ export interface TikTokConstructor { headers: Headers; verifyFp?: string; sessionList?: string[]; + throttleLimit?: number; + throttleInterval?: number; } export interface Hashtags {