Skip to content

Commit

Permalink
Merge pull request #4 from ai-forever/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@boomb0om
boomb0om authored Mar 21, 2024
2 parents 4d46a26 + ea37805 commit ca1f666
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 20 deletions.
22 changes: 12 additions & 10 deletions fsconnectors/asyncio/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,6 @@ async def mkdir(self, path: str) -> None:

async def copy(self, src_path: str, dst_path: str, recursive: bool = False) -> None:
if recursive:
src_path = src_path.rstrip('/') + '/'
dst_path = dst_path.rstrip('/') + '/'
src_bucket, src_key = self._split_path(src_path)
dst_bucket, dst_key = self._split_path(dst_path)
paths = await self.listdir(src_path, recursive)
Expand All @@ -135,7 +133,6 @@ async def move(self, src_path: str, dst_path: str, recursive: bool = False) -> N

async def remove(self, path: str, recursive: bool = False) -> None:
if recursive:
path = path.rstrip('/') + '/'
paths = await self.listdir(path, recursive)
for path in paths:
path_bucket, path_key = self._split_path(path)
Expand All @@ -154,20 +151,13 @@ async def listdir(self, path: str, recursive: bool = False) -> list[str]:

async def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
result = []
path = path.rstrip('/') + '/'
bucket, prefix = self._split_path(path)
paginator = self.client.get_paginator('list_objects')
if recursive:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
else:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
PaginationConfig={'PageSize': 1000})
async for item in paginator_result.search('CommonPrefixes'):
if item:
path = bucket + '/' + item.get('Prefix')
name = path.split('/')[-2]
if name:
result.append(FSEntry(name, path, 'dir'))
async for item in paginator_result.search('Contents'):
if item:
path = bucket + '/' + item.get('Key')
Expand All @@ -176,6 +166,18 @@ async def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
size = item.get('Size')
last_modified = item.get('LastModified')
result.append(FSEntry(name, path, 'file', size, last_modified))
paginator = self.client.get_paginator('list_objects')
if recursive:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
else:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
PaginationConfig={'PageSize': 1000})
async for item in paginator_result.search('CommonPrefixes'):
if item:
path = bucket + '/' + item.get('Prefix')
name = path.split('/')[-2]
if name:
result.append(FSEntry(name, path, 'dir'))
return result

async def upload_fileobj(
Expand Down
22 changes: 12 additions & 10 deletions fsconnectors/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,6 @@ def mkdir(self, path: str) -> None:
def copy(self, src_path: str, dst_path: str, recursive: bool = False) -> None:
client = self._get_client()
if recursive:
src_path = src_path.rstrip('/') + '/'
dst_path = dst_path.rstrip('/') + '/'
src_bucket, src_key = self._split_path(src_path)
dst_bucket, dst_key = self._split_path(dst_path)
paths = self.listdir(src_path, recursive)
Expand All @@ -116,7 +114,6 @@ def move(self, src_path: str, dst_path: str, recursive: bool = False) -> None:
def remove(self, path: str, recursive: bool = False) -> None:
client = self._get_client()
if recursive:
path = path.rstrip('/') + '/'
paths = self.listdir(path, recursive)
for path in paths:
path_bucket, path_key = self._split_path(path)
Expand All @@ -137,20 +134,13 @@ def listdir(self, path: str, recursive: bool = False) -> list[str]:
def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
client = self._get_client()
result = []
path = path.rstrip('/') + '/'
bucket, prefix = self._split_path(path)
paginator = client.get_paginator('list_objects')
if recursive:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
else:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
PaginationConfig={'PageSize': 1000})
for item in paginator_result.search('CommonPrefixes'):
if item:
path = bucket + '/' + item.get('Prefix')
name = path.split('/')[-2]
if name:
result.append(FSEntry(name, path, 'dir'))
for item in paginator_result.search('Contents'):
if item:
path = bucket + '/' + item.get('Key')
Expand All @@ -159,6 +149,18 @@ def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
size = item.get('Size')
last_modified = item.get('LastModified')
result.append(FSEntry(name, path, 'file', size, last_modified))
paginator = client.get_paginator('list_objects')
if recursive:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
else:
paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
PaginationConfig={'PageSize': 1000})
for item in paginator_result.search('CommonPrefixes'):
if item:
path = bucket + '/' + item.get('Prefix')
name = path.split('/')[-2]
if name:
result.append(FSEntry(name, path, 'dir'))
client.close()
return result

Expand Down

0 comments on commit ca1f666

Please sign in to comment.