throttle requests based on user parameters

drawrowfly · Aug 23, 2021 · 97dad6d · 97dad6d
1 parent 3715f1e
commit 97dad6d
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 0 deletions.
diff --git a/bin/cli.js b/bin/cli.js
@@ -27,6 +27,10 @@ const startScraper = async argv => {
         if (argv.historypath) {
             argv.historyPath = argv.historypath;
         }
+        if (argv.throttleinterval || argv.throttlelimit) {
+            argv.throttleLimit = argv.throttlelimit;
+            argv.throttleInterval = argv.throttleinterval;
+        }
         if (argv.file) {
             argv.input = argv.file;
         }
@@ -205,6 +209,12 @@ yargs
             default: process.env.SCRAPING_FROM_DOCKER ? '' : tmpdir(),
             describe: 'Set custom path where history file/files will be stored',
         },
+        throttlelimit: {
+            describe: 'Set custom maximum number of calls to TikTok within an interval.',
+        },
+        throttleinterval: {
+            describe: 'Set custom timespan for throttle-interval in milliseconds',
+        },
         remove: {
             alias: ['r'],
             default: '',

diff --git a/package.json b/package.json
@@ -18,6 +18,7 @@
         "jsdom": "^16.5.3",
         "json2csv": "4.5.1",
         "ora": "^4.0.2",
+        "p-throttle": "^4.1.1",
         "progress": "^2.0.3",
         "request": "^2.88.0",
         "request-promise": "^4.2.4",

diff --git a/src/core/TikTok.ts b/src/core/TikTok.ts
@@ -13,6 +13,7 @@ import { EventEmitter } from 'events';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import { forEachLimit } from 'async';
 import { URLSearchParams } from 'url';
+import pThrottle from 'p-throttle';
 import CONST from '../constant';
 import { sign, makeid } from '../helpers';
 
@@ -125,6 +126,8 @@ export class TikTokScraper extends EventEmitter {
 
     private store: string[];
 
+    private throttle: ReturnType<typeof pThrottle> | false;
+
     public cookieJar: CookieJar;
 
     constructor({
@@ -157,6 +160,8 @@ export class TikTokScraper extends EventEmitter {
         headers,
         verifyFp = '',
         sessionList = [],
+        throttleLimit,
+        throttleInterval,
     }: TikTokConstructor) {
         super();
         this.userIdStore = '';
@@ -221,6 +226,16 @@ export class TikTokScraper extends EventEmitter {
             bad: 0,
         };
         this.store = [];
+        this.throttle =
+            !!(throttleLimit && throttleInterval) &&
+            pThrottle({
+                limit: throttleLimit,
+                interval: throttleInterval,
+            });
+
+        if (this.throttle) {
+            this.request = <TikTokScraper['request']>this.throttle(this.request);
+        }
     }
 
     /**

diff --git a/src/types/TikTok.ts b/src/types/TikTok.ts
@@ -43,6 +43,8 @@ export interface Options {
     remove?: string;
     fileName?: string;
     historyPath?: string;
+    throttleLimit?: number;
+    throttleInterval?: number;
     timeout?: number;
     hdVideo?: boolean;
     randomUa?: boolean;
@@ -83,6 +85,8 @@ export interface TikTokConstructor {
     headers: Headers;
     verifyFp?: string;
     sessionList?: string[];
+    throttleLimit?: number;
+    throttleInterval?: number;
 }
 
 export interface Hashtags {