From b280ca819ab94341922c0e34c5513d50d5ae88a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Co=C5=9Fkun=20Deniz?= Date: Sun, 24 Dec 2023 12:37:54 +0300 Subject: [PATCH] Add new Chrome options, user agents, and simplify requirements --- requirements.txt | 32 ++------------------------------ utils.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6d73475..2f116ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,2 @@ -async-generator==1.10 -attrs==23.1.0 -certifi==2023.7.22 -chardet==4.0.0 -charset-normalizer==2.1.1 -click==8.1.3 -colorama==0.4.4 -configparser==5.0.2 -crayons==0.4.0 -exceptiongroup==1.1.3 -h11==0.14.0 -idna==2.10 -mypy-extensions==0.4.3 -outcome==1.2.0 -pathspec==0.10.1 -platformdirs==2.5.2 -PySocks==1.7.1 -random-user-agent==1.0.1 -requests==2.31.0 -selenium==4.9.1 -sniffio==1.3.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -trio==0.22.0 -trio-websocket==0.9.2 -typing_extensions==4.4.0 -undetected-chromedriver==3.5.3 -urllib3==2.0.7 -websockets==10.3 -wsproto==1.2.0 +selenium==4.16.0 +undetected-chromedriver==3.5.4 \ No newline at end of file diff --git a/utils.py b/utils.py index 5af9e0d..02f26b0 100644 --- a/utils.py +++ b/utils.py @@ -43,16 +43,29 @@ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5112.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5735.57 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.5735.57 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5735.57 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.5735.57 Mobile Safari/537.36", "Mozilla/5.0 (Linux; Android 10; SM-A205U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36", "Mozilla/5.0 (Linux; Android 10; LM-Q720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36", "Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/107.0.5304.101 Mobile/15E148 Safari/604.1", "Mozilla/5.0 (iPad; CPU OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/107.0.5304.101 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (iPad; CPU OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1" ] @@ -284,12 +297,27 @@ def create_webdriver( chrome_options.add_argument("--disable-popup-blocking") chrome_options.add_argument("--disable-notifications") chrome_options.add_argument("--ignore-ssl-errors") + chrome_options.add_argument("--ignore-certificate-errors") + chrome_options.add_argument("--disable-translate") + chrome_options.add_argument("--deny-permission-prompts") chrome_options.add_argument("--start-maximized") chrome_options.add_argument("--no-first-run") chrome_options.add_argument("--no-service-autorun") chrome_options.add_argument("--disable-blink-features=AutomationControlled") chrome_options.add_argument(f"--user-agent={user_agent_str}") + # disable these features for saving some proxy bandwidth + optimization_features = [ + "OptimizationGuideModelDownloading", + "OptimizationHintsFetching", + "OptimizationTargetPrediction", + "OptimizationHints", + "Translate", + "DownloadBubble", + "DownloadBubbleV2", + ] + chrome_options.add_argument(f"--disable-features={','.join(optimization_features)}") + # disable WebRTC IP tracking webrtc_preferences = { "webrtc.ip_handling_policy": "disable_non_proxied_udp",