diff --git a/aws_log_parser/aws/s3.py b/aws_log_parser/aws/s3.py index f109371..962a158 100644 --- a/aws_log_parser/aws/s3.py +++ b/aws_log_parser/aws/s3.py @@ -33,7 +33,7 @@ def read_key(self, bucket, key): print(f"Reading s3://{bucket}/{key}") contents = self.client.get_object(Bucket=bucket, Key=key) yield from FileIterator( - fileobj=BytesIO(contents["Body"].iter_lines()), + fileobj=BytesIO(contents["Body"].read()), gzipped=key.endswith(".gz"), ) diff --git a/test/test_interface.py b/test/test_interface.py index 751afb5..1c397cc 100644 --- a/test/test_interface.py +++ b/test/test_interface.py @@ -24,9 +24,7 @@ def paginate(self, **_): "Contents": [ { "Key": f"cloudfront-multiple.log{suffix}", - "LastModified": datetime.datetime( - 2021, 11, 28, 3, 31, 56, tzinfo=tzutc() - ), + "LastModified": datetime.datetime(2021, 11, 28, 3, 31, 56, tzinfo=tzutc()), "ETag": '"37c13f9a66a79c2b474356adaf5da1d0"', "Size": 2844, "StorageClass": "STANDARD", @@ -39,9 +37,12 @@ def paginate(self, **_): class MockStreamingFile: filename: str - def iter_lines(self): + def read(self): return open(self.filename, "rb").read() + def iter_lines(self): + yield from [line for line in open(self.filename, "rb")] + @dataclass class MockS3Client: @@ -104,9 +105,7 @@ def test_parse_s3_gzipped(monkeypatch, cloudfront_parser): def test_parse_url_s3(monkeypatch, cloudfront_parser): monkeypatch.setattr(S3Service, "client", MockS3Client()) - entries = cloudfront_parser.read_url( - "s3://aws-logs-test-data/cloudfront-multiple.log" - ) + entries = cloudfront_parser.read_url("s3://aws-logs-test-data/cloudfront-multiple.log") assert len(list(entries)) == 6