add test retries

scrapfly · Sep 23, 2024 · c2f5c9b · c2f5c9b
1 parent aca5eac
commit c2f5c9b
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 0 deletions.
diff --git a/nordstorm-scraper/pyproject.toml b/nordstorm-scraper/pyproject.toml
@@ -19,6 +19,7 @@ ruff = "^0.0.269"
 cerberus = "^1.3.4"
 pytest = "^7.3.1"
 pytest-asyncio = "^0.21.0"
+pytest-rerunfailures = "^14.0"
 
 [build-system]
 requires = ["poetry-core"]

diff --git a/nordstorm-scraper/test.py b/nordstorm-scraper/test.py
@@ -101,6 +101,7 @@ def validate_or_fail(item, validator):
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3, reruns_delay=30)
 async def test_product_scraping():
     products_data = await nordstorm.scrape_products(
         urls=[
@@ -116,6 +117,7 @@ async def test_product_scraping():
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3, reruns_delay=30)
 async def test_search_scraping():
     search_data = await nordstorm.scrape_search(
         url="https://www.nordstrom.com/sr?origin=keywordsearch&keyword=indigo", max_pages=3

diff --git a/realtorcom-scraper/pyproject.toml b/realtorcom-scraper/pyproject.toml
@@ -18,6 +18,7 @@ ruff = "^0.0.269"
 cerberus = "^1.3.4"
 pytest = "^7.3.1"
 pytest-asyncio = "^0.21.0"
+pytest-rerunfailures = "^14.0"
 
 [build-system]
 requires = ["poetry-core"]

diff --git a/realtorcom-scraper/test.py b/realtorcom-scraper/test.py
@@ -17,6 +17,7 @@ def validate_or_fail(item, validator):
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3, reruns_delay=30)
 async def test_property_scraping():
     url = "https://www.realtor.com/realestateandhomes-detail/12355-Attlee-Dr_Houston_TX_77077_M70330-35605"
     result = await realtorcom.scrape_property(url)
@@ -57,6 +58,7 @@ async def test_property_scraping():
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3, reruns_delay=30)
 async def test_search_scraping():
     result = await realtorcom.scrape_search("CA", "San-Francisco", max_pages=2)
     schema = {
@@ -71,6 +73,7 @@ async def test_search_scraping():
         validate_or_fail(item, validator)
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3, reruns_delay=30)
 async def test_feed_scraping():
     url = "https://www.realtor.com/realestateandhomes-detail/sitemap-rss-price/rss-price-ca.xml"
     result_feed = await realtorcom.scrape_feed(url)