fix: drop dirs param in s3 listdir/scandir

ai-forever · Mar 21, 2024 · ea37805 · ea37805
1 parent 7e3982e
commit ea37805
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 32 deletions.
diff --git a/fsconnectors/asyncio/s3.py b/fsconnectors/asyncio/s3.py
@@ -141,15 +141,15 @@ async def remove(self, path: str, recursive: bool = False) -> None:
             bucket, key = self._split_path(path)
             await self.client.delete_object(Bucket=bucket, Key=key)
 
-    async def listdir(self, path: str, recursive: bool = False, dirs: bool = False) -> list[str]:
-        entries = await self.scandir(path, recursive, dirs)
+    async def listdir(self, path: str, recursive: bool = False) -> list[str]:
+        entries = await self.scandir(path, recursive)
         if recursive:
             result = [entry.path for entry in entries]
         else:
             result = [entry.name for entry in entries]
         return result
 
-    async def scandir(self, path: str, recursive: bool = False, dirs: bool = False) -> list[FSEntry]:
+    async def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
         result = []
         bucket, prefix = self._split_path(path)
         paginator = self.client.get_paginator('list_objects')
@@ -166,19 +166,18 @@ async def scandir(self, path: str, recursive: bool = False, dirs: bool = False)
                     size = item.get('Size')
                     last_modified = item.get('LastModified')
                     result.append(FSEntry(name, path, 'file', size, last_modified))
-        if dirs:
-            paginator = self.client.get_paginator('list_objects')
-            if recursive:
-                paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
-            else:
-                paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
-                                                      PaginationConfig={'PageSize': 1000})
-            async for item in paginator_result.search('CommonPrefixes'):
-                if item:
-                    path = bucket + '/' + item.get('Prefix')
-                    name = path.split('/')[-2]
-                    if name:
-                        result.append(FSEntry(name, path, 'dir'))
+        paginator = self.client.get_paginator('list_objects')
+        if recursive:
+            paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
+        else:
+            paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
+                                                  PaginationConfig={'PageSize': 1000})
+        async for item in paginator_result.search('CommonPrefixes'):
+            if item:
+                path = bucket + '/' + item.get('Prefix')
+                name = path.split('/')[-2]
+                if name:
+                    result.append(FSEntry(name, path, 'dir'))
         return result
 
     async def upload_fileobj(

diff --git a/fsconnectors/s3.py b/fsconnectors/s3.py
@@ -123,15 +123,15 @@ def remove(self, path: str, recursive: bool = False) -> None:
             client.delete_object(Bucket=bucket, Key=key)
         client.close()
 
-    def listdir(self, path: str, recursive: bool = False, dirs: bool = False) -> list[str]:
-        entries = self.scandir(path, recursive, dirs)
+    def listdir(self, path: str, recursive: bool = False) -> list[str]:
+        entries = self.scandir(path, recursive)
         if recursive:
             result = [entry.path for entry in entries]
         else:
             result = [entry.name for entry in entries]
         return result
 
-    def scandir(self, path: str, recursive: bool = False, dirs: bool = False) -> list[FSEntry]:
+    def scandir(self, path: str, recursive: bool = False) -> list[FSEntry]:
         client = self._get_client()
         result = []
         bucket, prefix = self._split_path(path)
@@ -149,19 +149,18 @@ def scandir(self, path: str, recursive: bool = False, dirs: bool = False) -> lis
                     size = item.get('Size')
                     last_modified = item.get('LastModified')
                     result.append(FSEntry(name, path, 'file', size, last_modified))
-        if dirs:
-            paginator = client.get_paginator('list_objects')
-            if recursive:
-                paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
-            else:
-                paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
-                                                      PaginationConfig={'PageSize': 1000})
-            for item in paginator_result.search('CommonPrefixes'):
-                if item:
-                    path = bucket + '/' + item.get('Prefix')
-                    name = path.split('/')[-2]
-                    if name:
-                        result.append(FSEntry(name, path, 'dir'))
+        paginator = client.get_paginator('list_objects')
+        if recursive:
+            paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, PaginationConfig={'PageSize': 1000})
+        else:
+            paginator_result = paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter='/',
+                                                  PaginationConfig={'PageSize': 1000})
+        for item in paginator_result.search('CommonPrefixes'):
+            if item:
+                path = bucket + '/' + item.get('Prefix')
+                name = path.split('/')[-2]
+                if name:
+                    result.append(FSEntry(name, path, 'dir'))
         client.close()
         return result