Skip to content
This repository has been archived by the owner on Oct 25, 2024. It is now read-only.

Commit

Permalink
Add new Chrome options, user agents, and simplify requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
@coskundeniz
coskundeniz committed Dec 24, 2023
1 parent 539739a commit b280ca8
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 31 deletions.
32 changes: 2 additions & 30 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,2 @@
async-generator==1.10
attrs==23.1.0
certifi==2023.7.22
chardet==4.0.0
charset-normalizer==2.1.1
click==8.1.3
colorama==0.4.4
configparser==5.0.2
crayons==0.4.0
exceptiongroup==1.1.3
h11==0.14.0
idna==2.10
mypy-extensions==0.4.3
outcome==1.2.0
pathspec==0.10.1
platformdirs==2.5.2
PySocks==1.7.1
random-user-agent==1.0.1
requests==2.31.0
selenium==4.9.1
sniffio==1.3.0
sortedcontainers==2.4.0
tomli==2.0.1
trio==0.22.0
trio-websocket==0.9.2
typing_extensions==4.4.0
undetected-chromedriver==3.5.3
urllib3==2.0.7
websockets==10.3
wsproto==1.2.0
selenium==4.16.0
undetected-chromedriver==3.5.4
30 changes: 29 additions & 1 deletion utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,29 @@
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5112.79 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5735.57 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; SM-N960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.5735.57 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.5735.57 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.5735.57 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; SM-A205U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; LM-Q720) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 10; LM-X420) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.105 Mobile Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/107.0.5304.101 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPad; CPU OS 16_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/107.0.5304.101 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPad; CPU OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.6099.119 Mobile/15E148 Safari/604.1"
]


Expand Down Expand Up @@ -284,12 +297,27 @@ def create_webdriver(
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument("--disable-notifications")
chrome_options.add_argument("--ignore-ssl-errors")
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--disable-translate")
chrome_options.add_argument("--deny-permission-prompts")
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--no-first-run")
chrome_options.add_argument("--no-service-autorun")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_argument(f"--user-agent={user_agent_str}")

# disable these features for saving some proxy bandwidth
optimization_features = [
"OptimizationGuideModelDownloading",
"OptimizationHintsFetching",
"OptimizationTargetPrediction",
"OptimizationHints",
"Translate",
"DownloadBubble",
"DownloadBubbleV2",
]
chrome_options.add_argument(f"--disable-features={','.join(optimization_features)}")

# disable WebRTC IP tracking
webrtc_preferences = {
"webrtc.ip_handling_policy": "disable_non_proxied_udp",
Expand Down

0 comments on commit b280ca8

Please sign in to comment.