Refactor: Replace deprecated request package by got (#563)

* Move debug contexts params to config (#552) * build: replace deprecated dependency with got package * build: lock file * feat: implement usage of got package for requests * build: adjust lock file
itteco · Nov 13, 2024 · bc935b6 · bc935b6
1 parent b608d28
commit bc935b6
Show file tree

Hide file tree

Showing 7 changed files with 744 additions and 797 deletions.
diff --git a/docs/WRITE-A-PLUGIN.md b/docs/WRITE-A-PLUGIN.md
@@ -190,14 +190,9 @@ Example:
         },
         {
             geturls: function(cb) {
-                var request = require('request');
-                request({
-                    url: 'https://api.domain.com/items',
-                    json: true
-                }, function(error, body, data) {
-                    if (error) {
-                        return cb(error);
-                    }
+                got('https://api.domain.com/items', { responseType: 'json' })
+                .then(response => {
+                    const data = response.body;
                     if (!data || !data.urls) {
                         return cb('No urls in API data');
                     }

diff --git a/lib/whitelist.js b/lib/whitelist.js
@@ -5,7 +5,7 @@
     import * as _ from 'underscore';
     import * as utils from './utils.js';
     import log from '../logging.js';
-    import request from 'request';
+    import got from 'got';
 
     import { fileURLToPath } from 'url';
     import { dirname } from 'path';
@@ -356,38 +356,33 @@
             log("Loading domains list from " + CONFIG.WHITELIST_URL);
 
             var options = {
-                uri: CONFIG.WHITELIST_URL,
-                json: true,
-                qs: {
+                url: CONFIG.WHITELIST_URL,
+                responseType: 'json',
+                searchParams: {
                     domain: CONFIG.baseAppUrl && CONFIG.baseAppUrl.replace(/.+\/\//, ''),
                     v: CONFIG.VERSION
                 },
                 headers: {
                     // Prevent caching.
                     'Cache-Control': 'no-cache'
                 },
-                // TODO: remove in helix-fetch
-                gzip: true
             };
-
+ 
             if (whitelistLastModified) {
                 options.headers['If-Modified-Since'] = whitelistLastModified;
             }
 
-            request(utils.prepareRequestOptions(options, {skipPlugins: true}), function(error, r, newWhitelist) {
+            got(options).then(response => {
+                const newWhitelist = response.body;
 
-                if (error) {
-                    console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : ' + error);
-                } else if (r.statusCode === 500) {
+                if (response.statusCode === 500) {
                     console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : ' + newWhitelist);
-                } else if (r.statusCode === 304) {
+                } else if (response.statusCode === 304) {
                     log('Whitelist respond: 304 (not modified)');
                 } else if (!newWhitelist || typeof newWhitelist === 'string') {
                     console.error('Error loading domains list from ' + CONFIG.WHITELIST_URL + ' : incorrect data: ' + newWhitelist);
                 } else {
-
-                    whitelistLastModified = r.headers['last-modified'];
-
+                    whitelistLastModified = response.headers['last-modified'];
                     applyParsedWhitelist(newWhitelist);
                 }
 

diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
@@ -1,6 +1,6 @@
 import * as _ from 'underscore';
 import FeedParser from 'feedparser';
-import request from 'request';
+import got from 'got';
 import * as async from 'async';
 import * as url from 'url';
 import { PageTestLog, TestUrlsSet, PluginTest } from './models.js';
@@ -50,35 +50,32 @@ export function sendQANotification(logEntry, data) {
             message += " - " + errors;
         }
 
-        request({
-            uri: CONFIG.SLACK_WEBHOOK_FOR_QA,
-            method: 'POST',
-            json: true,
-            body: {
-                "parse": "none",
-                "channel": CONFIG.SLACK_CHANNEL_FOR_QA,
-                "username": SLACK_USERNAME,
-                "text": previewMessage,
-                "blocks": [
+        got.post(CONFIG.SLACK_WEBHOOK_FOR_QA, {
+            json: {
+                parse: "none",
+                channel: CONFIG.SLACK_CHANNEL_FOR_QA,
+                username: SLACK_USERNAME,
+                text: previewMessage,
+                blocks: [
                     {
-                        "type": "section",
-                        "text": {
-                            "type": "mrkdwn",
-                            "text": message  // Message: [domain.com] Failed - errors.
+                        type: "section",
+                        text: {
+                            type: "mrkdwn",
+                            text: message
                         }
                     }
                 ],
-                "attachments": [
+                attachments: [
                     {
-                        "blocks": [{
-                            "type": "section",
-                            "text": {
-                                "type": "mrkdwn",
-                                "verbatim": true,
-                                "text": "`<" + (CONFIG.QA_BASE_URL || baseAppUrl) + "/debug?uri=" + encodeURIComponent(logEntry.url) + "|debug>` " + logEntry.url.replace(/^https?:\/\//, '')    // Debug link.
+                        blocks: [{
+                            type: "section",
+                            text: {
+                                type: "mrkdwn",
+                                verbatim: true,
+                                text: "`<" + (CONFIG.QA_BASE_URL || baseAppUrl) + "/debug?uri=" + encodeURIComponent(logEntry.url) + "|debug>` " + logEntry.url.replace(/^https?:\/\//, '')
                             }
                         }],
-                        "color": COLORS[data.color]
+                        color: COLORS[data.color]
                     }
                 ]
             }
@@ -272,43 +269,43 @@ export function fetchFeedUrls(feedUrl, options, cb) {
         cb(error, urls);
     };
 
-    request({
-        uri: feedUrl,
-        agentOptions: {
+
+    got.stream(feedUrl, {
+        https: {
             rejectUnauthorized: false
         }
     })
-        .pipe(new FeedParser({addmeta: false}))
-        .on('error', function(error) {
-            _cb(error);
-        })
-        .on('readable', function () {
-            var stream = this, item;
-            while (item = stream.read()) {
+    .pipe(new FeedParser({ addmeta: false }))
+    .on('error', function(error) {
+        _cb(error);
+    })
+    .on('readable', function () {
+        var stream = this, item;
+        while (item = stream.read()) {
 
-                if (urls.length < MAX_FEED_URLS) {
+            if (urls.length < MAX_FEED_URLS) {
 
-                    var url = item.origlink || item.link;
+                var url = item.origlink || item.link;
 
-                    if (options.getUrl) {
-                        url = options.getUrl(url);
-                    }
+                if (options.getUrl) {
+                    url = options.getUrl(url);
+                }
 
-                    if (!url) {
-                        return;
-                    }
+                if (!url) {
+                    return;
+                }
 
-                    urls.push(url);
+                urls.push(url);
 
-                    if (MAX_FEED_URLS == urls.length) {
-                        _cb();
-                    }
+                if (MAX_FEED_URLS === urls.length) {
+                    _cb();
                 }
             }
-        })
-        .on('end', function() {
-            _cb();
-        });
+        }
+    })
+    .on('end', function() {
+        _cb();
+    });
 };
 
 export function fetchUrlsByPageOnFeed(pageWithFeed, otpions, cb) {

diff --git a/package.json b/package.json
@@ -30,6 +30,7 @@
         "ejs": "3.1.10",
         "entities": "1.1.2",
         "express": "^4.19.0",
+        "got": "^14.4.4",
         "graceful-cluster": "^0.0.5",
         "htmlparser2": "^7.2.0",
         "iconv-lite": "^0.6.3",
@@ -44,7 +45,6 @@
         "readabilitySAX": "1.6.1",
         "redis": "^4.6.14",
         "redis-clustr": "1.7.0",
-        "request": "^2.88.2",
         "sax": "^1.2.4",
         "send": "^0.18.0",
         "underscore": "^1.13.6"

diff --git a/plugins/domains/dailymotion.com/dailymotion.com.js b/plugins/domains/dailymotion.com/dailymotion.com.js
@@ -1,5 +1,5 @@
 import * as querystring from 'querystring';
-import request from 'request';
+import got from 'got';
 
 export default {
 
@@ -53,21 +53,17 @@ export default {
 
     tests: [{
         getUrls: function(cb) {
-            request({
-                url: 'https://api.dailymotion.com/videos',
-                json: true
-            }, function(error, body, data) {
-                if (error) {
-                    return cb(error);
-                }
+            got('https://api.dailymotion.com/videos', { responseType: 'json' })
+            .then(response => {
+                const data = response.body;
                 if (!data || !data.list) {
                     return cb('No videos list in API data');
                 }
                 cb(null, data.list.slice(0, 10).map(function(item) {
                     return 'https://www.dailymotion.com/video/' + item.id;
                 }));
-            });
-
+            })
+            .catch(error => cb(error));
         }
     }, {
         skipMixins: ["video", "og-description", "og-image", "canonical"],

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml