Skip to content

Commit

Permalink
bump up request timeout values
Browse files Browse the repository at this point in the history
if necessary, these can be extracted to runtime arguments
  • Loading branch information
alexdunnjpl committed Sep 12, 2023
1 parent a0d0b2a commit ecb261f
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions src/pds/registrysweepers/utils/db/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ def query_registry_db(
"""

scroll_keepalive = f"{scroll_keepalive_minutes}m"
request_timeout = 20
query_id = get_random_hex_id() # This is just used to differentiate queries during logging
log.info(f"Initiating query with id {query_id}: {query}")

Expand All @@ -51,6 +52,7 @@ def fetch_func():
index=index_name,
body=query,
scroll=scroll_keepalive,
request_timeout=request_timeout,
size=page_size,
_source_includes=_source.get("includes", []), # TODO: Break out from the enclosing _source object
_source_excludes=_source.get("excludes", []), # TODO: Break out from the enclosing _source object
Expand All @@ -59,7 +61,7 @@ def fetch_func():
else:

def fetch_func(_scroll_id: str = scroll_id):
return client.scroll(scroll_id=_scroll_id, scroll=scroll_keepalive)
return client.scroll(scroll_id=_scroll_id, scroll=scroll_keepalive, request_timeout=request_timeout)

results = retry_call(
fetch_func,
Expand Down Expand Up @@ -174,7 +176,8 @@ def update_as_statements(update: Update) -> Iterable[str]:
def _write_bulk_updates_chunk(client: OpenSearch, index_name: str, bulk_updates: Iterable[str]):
bulk_data = "\n".join(bulk_updates) + "\n"

response_content = client.bulk(index=index_name, body=bulk_data)
request_timeout = 60
response_content = client.bulk(index=index_name, body=bulk_data, request_timeout=request_timeout)

if response_content.get("errors"):
warn_types = {"document_missing_exception"} # these types represent bad data, not bad sweepers behaviour
Expand Down

0 comments on commit ecb261f

Please sign in to comment.