From c818cf2166252e7e6e30bf4ed12caccb1dc26c85 Mon Sep 17 00:00:00 2001 From: Robert Kenny Date: Mon, 6 Jan 2025 15:15:15 +0000 Subject: [PATCH 1/2] use localstack in storage tests --- storage/docker-compose.yml | 18 ++++-------------- .../weco/storage/providers/s3/S3Errors.scala | 7 +++++++ .../dynamo/DynamoHybridStoreTestCases.scala | 4 ++-- .../storage/store/s3/S3StreamStoreTest.scala | 9 +++++---- .../storage/store/s3/S3TypedStoreTest.scala | 4 ++-- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/storage/docker-compose.yml b/storage/docker-compose.yml index fc20c6cd..863366d0 100644 --- a/storage/docker-compose.yml +++ b/storage/docker-compose.yml @@ -5,19 +5,9 @@ services: ports: - "45678:8000" s3: - image: "zenko/cloudserver:8.1.8" + image: "public.ecr.aws/localstack/localstack:4.0.0" environment: - - "S3BACKEND=mem" + - SERVICES=s3 + - ALLOW_NONSTANDARD_REGIONS=1 ports: - - "33333:8000" - - # We've seen flakiness when this container doesn't start fast enough, - # and the first few tests to interact with S3 fail. This uses - # docker-compose healthchecks to check the container is returning - # a 401 Unauthorized before continuing. - # See https://docs.docker.com/compose/compose-file/#/healthcheck - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:33333"] - interval: 2s - timeout: 10s - retries: 5 + - "33333:4566" diff --git a/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala b/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala index 97f704dc..02899985 100644 --- a/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala +++ b/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala @@ -28,6 +28,10 @@ object S3Errors { if exc.getMessage.startsWith("The specified bucket is not valid") => StoreReadError(exc) + case exc: S3Exception + if exc.getMessage.startsWith("The specified bucket does not exist") => + StoreReadError(exc) + case exc: SdkClientException if exc.getMessage.startsWith("Unable to execute HTTP request") => new StoreReadError(exc) with RetryableError @@ -54,6 +58,9 @@ object S3Errors { case exc: S3Exception if exc.getMessage.startsWith("Object key is too long") => InvalidIdentifierFailure(exc) + case exc: S3Exception + if exc.getMessage.startsWith("Your key is too long") => + InvalidIdentifierFailure(exc) case exc => StoreWriteError(exc) } diff --git a/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala b/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala index 50ae734a..cda0337e 100644 --- a/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala +++ b/storage/src/test/scala/weco/storage/store/dynamo/DynamoHybridStoreTestCases.scala @@ -171,7 +171,7 @@ trait DynamoHybridStoreTestCases[ value shouldBe a[StoreWriteError] value.e.getMessage should startWith( - "The specified bucket is not valid") + "The specified bucket does not exist") } } } @@ -206,7 +206,7 @@ trait DynamoHybridStoreTestCases[ val value = result.left.value value shouldBe a[InvalidIdentifierFailure] - value.e.getMessage should startWith("Object key is too long") + value.e.getMessage should include("key is too long") } } } diff --git a/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala index 07611a4e..6fc3d0e6 100644 --- a/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala +++ b/storage/src/test/scala/weco/storage/store/s3/S3StreamStoreTest.scala @@ -59,10 +59,11 @@ class S3StreamStoreTest withStoreImpl(initialEntries = Map.empty) { store => val invalidLocation = createS3ObjectLocationWith(createInvalidBucket) val err = store.get(invalidLocation).left.value - err shouldBe a[StoreReadError] + err shouldBe a[DoesNotExistError] err.e shouldBe a[S3Exception] - err.e.getMessage should startWith("The specified bucket is not valid") + err.e.getMessage should startWith( + "The specified bucket does not exist") } } } @@ -102,7 +103,7 @@ class S3StreamStoreTest val err = result.e err shouldBe a[S3Exception] - err.getMessage should startWith("The specified bucket is not valid") + err.getMessage should startWith("The specified bucket does not exist") } } @@ -121,7 +122,7 @@ class S3StreamStoreTest val value = store.put(id)(entry).left.value value shouldBe a[InvalidIdentifierFailure] - value.e.getMessage should startWith("Object key is too long") + value.e.getMessage should include("key is too long") } } } diff --git a/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala b/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala index 864a9af0..7f20759a 100644 --- a/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala +++ b/storage/src/test/scala/weco/storage/store/s3/S3TypedStoreTest.scala @@ -61,7 +61,7 @@ class S3TypedStoreTest describe("S3TypedStore") { it("errors if the object key is too long") { withLocalS3Bucket { bucket => - // Maximum length of an s3 key is 1024 bytes as of 25/06/2019 + // Maximum length of a s3 key is 1024 bytes as of 25/06/2019 // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html val location = S3ObjectLocation( bucket = bucket.name, @@ -74,7 +74,7 @@ class S3TypedStoreTest val value = store.put(location)(entry).left.value value shouldBe a[InvalidIdentifierFailure] - value.e.getMessage should startWith("Object key is too long") + value.e.getMessage should include("key is too long") } } } From b2c73b7b66bc3f72a269f96e0e6dfe1d84ba7f4a Mon Sep 17 00:00:00 2001 From: Github on behalf of Wellcome Collection Date: Mon, 6 Jan 2025 15:26:30 +0000 Subject: [PATCH 2/2] Apply auto-formatting rules --- .../src/main/scala/weco/storage/providers/s3/S3Errors.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala b/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala index 02899985..77b9f104 100644 --- a/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala +++ b/storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala @@ -29,7 +29,7 @@ object S3Errors { StoreReadError(exc) case exc: S3Exception - if exc.getMessage.startsWith("The specified bucket does not exist") => + if exc.getMessage.startsWith("The specified bucket does not exist") => StoreReadError(exc) case exc: SdkClientException @@ -59,7 +59,7 @@ object S3Errors { if exc.getMessage.startsWith("Object key is too long") => InvalidIdentifierFailure(exc) case exc: S3Exception - if exc.getMessage.startsWith("Your key is too long") => + if exc.getMessage.startsWith("Your key is too long") => InvalidIdentifierFailure(exc) case exc => StoreWriteError(exc)