Skip to content

Commit

Permalink
Ensure consistent use of versions during pagination (#591)
Browse files Browse the repository at this point in the history
  • Loading branch information
rlamy authored Nov 29, 2023
1 parent 4422f45 commit 20959bb
Showing 1 changed file with 4 additions and 7 deletions.
11 changes: 4 additions & 7 deletions gcsfs/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,7 @@ async def _do_list_objects(
bucket, _path, generation = self.split_path(path)
_path = "" if not _path else _path.rstrip("/") + "/"
prefix = f"{_path}{prefix}" or None
versions = bool(versions or generation)

# Page size of 5000 is officially supported across GCS.
default_page_size = 5000
Expand Down Expand Up @@ -628,7 +629,6 @@ async def _do_list_objects(
delimiter=delimiter,
prefix=prefix,
versions=versions,
generation=generation,
page_size=default_page_size,
)

Expand All @@ -642,12 +642,11 @@ async def _do_list_objects(
end_offset=None,
prefix=prefix,
versions=versions,
generation=generation,
page_size=default_page_size,
)

async def _concurrent_list_objects_helper(
self, items, bucket, delimiter, prefix, versions, generation, page_size
self, items, bucket, delimiter, prefix, versions, page_size
):
"""
Lists objects using coroutines, using the object names from the inventory
Expand Down Expand Up @@ -696,7 +695,6 @@ async def _concurrent_list_objects_helper(
end_offset=end_offsets[i],
prefix=prefix,
versions=versions,
generation=generation,
page_size=page_size,
)
for i in range(0, len(start_offsets))
Expand All @@ -722,7 +720,6 @@ async def _sequential_list_objects_helper(
end_offset,
prefix,
versions,
generation,
page_size,
):
"""
Expand All @@ -742,7 +739,7 @@ async def _sequential_list_objects_helper(
endOffset=end_offset,
maxResults=page_size,
json_out=True,
versions="true" if versions or generation else None,
versions="true" if versions else None,
)

prefixes.extend(page.get("prefixes", []))
Expand All @@ -761,7 +758,7 @@ async def _sequential_list_objects_helper(
maxResults=page_size,
pageToken=next_page_token,
json_out=True,
versions="true" if generation else None,
versions="true" if versions else None,
)

assert page["kind"] == "storage#objects"
Expand Down

0 comments on commit 20959bb

Please sign in to comment.