Skip to content

Commit

Permalink
Avoid endless retry
Browse files Browse the repository at this point in the history
  • Loading branch information
jbothma committed Nov 15, 2023
1 parent f9fcfee commit 84d29e7
Showing 1 changed file with 11 additions and 3 deletions.
14 changes: 11 additions & 3 deletions nomenklatura/enrich/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def http_post_json_cached(
cache_key: str,
json: Any,
cache_days: Optional[int] = None,
retry: int = 3,
) -> Any:
cache_days_ = self.cache_days if cache_days is None else cache_days
resp_data = self.cache.get_json(cache_key, max_age=cache_days_)
Expand All @@ -113,9 +114,16 @@ def http_post_json_cached(
if rex.response is not None and rex.response.status_code in (401, 403):
raise EnrichmentAbort("Authorization failure: %s" % url) from rex
if rex.response is not None and rex.response.status_code == 429:
log.info("Rate limit exceeded. Sleeping for 60s.")
time.sleep(61)
return self.http_post_json_cached(url, cache_key, json, cache_days)
if retry > 0:
log.info("Rate limit exceeded. Sleeping for 60s.")
time.sleep(61)
return self.http_post_json_cached(
url, cache_key, json, cache_days, retry - 1
)
else:
raise EnrichmentException(
"Rate limit exceeded and out of retries: %s" % url
) from rex
msg = "HTTP POST failed [%s]: %s" % (url, rex)
raise EnrichmentException(msg) from rex
resp_data = resp.json()
Expand Down

0 comments on commit 84d29e7

Please sign in to comment.