Upgrade scrapy to version 2.2.0 (#47)

* Remove custom context factory Update scrapy to latest version * Fix Changelog after update
meilisearch · Jul 7, 2020 · 2e14dbe · 2e14dbe
1 parent 8e34402
commit 2e14dbe
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 27 deletions.
diff --git a/Pipfile b/Pipfile
@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-Scrapy = "==1.7.*"
+Scrapy = "==2.2.0"
 selenium = "==3.141.0"
 pytest = "==5.4.3"
 meilisearch = "==0.11.2"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/scraper/src/index.py b/scraper/src/index.py
@@ -16,7 +16,6 @@
 from .custom_downloader_middleware import CustomDownloaderMiddleware
 from .custom_dupefilter import CustomDupeFilter
 from .config.browser_handler import BrowserHandler
-from .scrapy_patch import CustomContextFactory
 
 try:
     # disable boto (S3 download)
@@ -46,7 +45,6 @@ def run_config(config):
 
     root_module = 'src.' if __name__ == '__main__' else 'scraper.src.'
     DOWNLOADER_MIDDLEWARES_PATH = root_module + 'custom_downloader_middleware.' + CustomDownloaderMiddleware.__name__
-    DOWNLOADER_CLIENTCONTEXTFACTORY = root_module + 'scrapy_patch.' + CustomContextFactory.__name__
     DUPEFILTER_CLASS_PATH = root_module + 'custom_dupefilter.' + CustomDupeFilter.__name__
 
     headers = {
@@ -78,7 +76,6 @@ def run_config(config):
         'USER_AGENT': config.user_agent,
         'DOWNLOADER_MIDDLEWARES': {DOWNLOADER_MIDDLEWARES_PATH: 900},
         # Need to be > 600 to be after the redirectMiddleware
-        'DOWNLOADER_CLIENTCONTEXTFACTORY': DOWNLOADER_CLIENTCONTEXTFACTORY,
         'DUPEFILTER_USE_ANCHORS': config.use_anchors,
         # Use our custom dupefilter in order to be scheme agnostic regarding link provided
         'DUPEFILTER_CLASS': DUPEFILTER_CLASS_PATH,

diff --git a/scraper/src/scrapy_patch.py b/scraper/src/scrapy_patch.py