test

smart/core.py

@@ -128,7 +128,7 @@ async def start(self):
 
             if resp is None:
                 # let the_downloader can be scheduled, test 0.001-0.0006 is better
-                await asyncio.sleep(0.005)
+                await asyncio.sleep(0.0005)
                 continue
 
             custome_callback = resp.request.callback

smart/downloader.py

@@ -33,6 +33,7 @@ class AioHttpDown(BaseDown):
 
     async def fetch(self, request: Request) -> Response:
         session = None
+        resp = None
         try:
             session = request.session or aiohttp.ClientSession()
             resp = await session.request(request.method,
@@ -53,8 +54,11 @@ async def fetch(self, request: Request) -> Response:
                                 cookies=resp.cookies
                                 )
         finally:
+            if resp:
+                resp.release()
             if request.session is None and session:
                 await session.close()
+
         return response
 
 

spiders/govs.py

@@ -5,8 +5,11 @@
 # Date:      2021/1/6
 # Desc:      there is a python file description
 # ------------------------------------------------------------------
+import asyncio
 import traceback
 
+import aiohttp
+
 from smart.field import HtmlField
 from smart.item import Item
 from smart.request import Request
@@ -27,6 +30,7 @@ class GovsSpider(Spider):
         "http://www.nea.gov.cn/policy/jd.htm"
     ]
 
+
     def parse(self, response: Response):
         selects_detail_urls = response.xpath(
             '//*[@class="list"]//li//a/@href').getall()
@@ -45,3 +49,5 @@ def parse_detail(self, response):
 
     def on_exception_occured(self, e: Exception):
         print(e)
+
+