Merge branch 'main' into update/non_aws

.github/workflows/release.yml

@@ -126,11 +126,13 @@ jobs:
           mkdir ~/.sbt
           echo "${{ secrets.SONATYPE_CREDENTIALS }}" > ~/.sbt/sonatype.credentials
       - name: Setup JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: 17
           cache: sbt
+      - name: Setup sbt
+        uses: sbt/[email protected]
       - name: Publish to Sonatype
         run: |
           ARTIFACT_NAME="${{ matrix.service }}_2.12"

CHANGELOG.md

@@ -1,5 +1,14 @@
 # CHANGELOG
 
+## v32.43.4 - 2025-01-07
+
+This is a change to the build process to fix a bug in the way that the `sbt` build is run.
+
+## v32.43.3 - 2025-01-06
+
+Update ElasticBuilder to generate and accept config case classes to enable 
+pushing typesafe config to the edge of consuming services.
+
 ## v32.43.2 - 2024-10-07
 
 S3StreamWritable bug fix (replace `read` by `readNBytes`).

build.sbt

@@ -1,4 +1,4 @@
-val projectVersion = "32.43.2"
+val projectVersion = "32.43.4"
 
 Global / excludeLintKeys += composeNoBuild
 

elasticsearch_typesafe/src/main/scala/weco/elasticsearch/typesafe/ElasticBuilder.scala

@@ -3,11 +3,35 @@ package weco.elasticsearch.typesafe
 import com.sksamuel.elastic4s.ElasticClient
 import com.typesafe.config.Config
 import weco.elasticsearch.ElasticClientBuilder
-import weco.typesafe.config.builders.EnrichConfig._
+
+sealed trait ElasticConfig {
+  val host: String
+  val port: Int
+  val protocol: String
+}
+
+case class ElasticConfigUsernamePassword(
+  host: String,
+  port: Int,
+  protocol: String,
+  username: String,
+  password: String
+) extends ElasticConfig
+
+case class ElasticConfigApiKey(
+  host: String,
+  port: Int,
+  protocol: String,
+  apiKey: String
+) extends ElasticConfig
 
 object ElasticBuilder {
-  def buildElasticClient(config: Config,
-                         namespace: String = ""): ElasticClient = {
+  import weco.typesafe.config.builders.EnrichConfig._
+
+  def buildElasticClientConfig(
+    config: Config,
+    namespace: String = ""
+  ): ElasticConfig = {
     val hostname = config.requireString(s"es.$namespace.host")
     val port = config
       .getIntOption(s"es.$namespace.port")
@@ -22,7 +46,7 @@ object ElasticBuilder {
       config.getStringOption(s"es.$namespace.apikey")
     ) match {
       case (Some(username), Some(password), None) =>
-        ElasticClientBuilder.create(
+        ElasticConfigUsernamePassword(
           hostname,
           port,
           protocol,
@@ -31,10 +55,37 @@ object ElasticBuilder {
         )
       // Use an API key if specified, even if username/password are also present
       case (_, _, Some(apiKey)) =>
-        ElasticClientBuilder.create(hostname, port, protocol, apiKey)
+        ElasticConfigApiKey(hostname, port, protocol, apiKey)
       case _ =>
         throw new Throwable(
-          s"You must specify username and password, or apikey, in the 'es.$namespace' config")
+          s"You must specify username and password, or apikey, in the 'es.$namespace' config"
+        )
     }
   }
+
+  def buildElasticClient(config: ElasticConfig): ElasticClient =
+    config match {
+      case ElasticConfigUsernamePassword(
+          hostname,
+          port,
+          protocol,
+          username,
+          password
+          ) =>
+        ElasticClientBuilder.create(
+          hostname,
+          port,
+          protocol,
+          username,
+          password
+        )
+      case ElasticConfigApiKey(hostname, port, protocol, apiKey) =>
+        ElasticClientBuilder.create(hostname, port, protocol, apiKey)
+    }
+
+  def buildElasticClient(
+    config: Config,
+    namespace: String = ""
+  ): ElasticClient =
+    buildElasticClient(buildElasticClientConfig(config, namespace))
 }

elasticsearch_typesafe/src/test/scala/weco/elasticsearch/typesafe/ElasticBuilderTest.scala

@@ -3,45 +3,116 @@ package weco.elasticsearch.typesafe
 import com.typesafe.config.ConfigFactory
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
+import weco.fixtures.RandomGenerators
 
-class ElasticBuilderTest extends AnyFunSpec with Matchers {
-  it("builds a client when a username and password are specified") {
-    val config = ConfigFactory.parseString(
-      """
-        |es.test.host = test.elastic
-        |es.test.username = test-username
-        |es.test.password = test-password
-        |""".stripMargin
-    )
+class ElasticBuilderTest
+    extends AnyFunSpec
+    with Matchers
+    with RandomGenerators {
 
-    noException shouldBe thrownBy(
-      ElasticBuilder.buildElasticClient(config, "test")
+  val List(namespace, host, username, password, apiKey) =
+    1.to(5).map(_ => randomAlphanumeric()).toList
+  val defaultPort = 9200
+  val defaultProtocol = "http"
+
+  describe("when username and password are specified") {
+    val usernamePasswordConfig = ConfigFactory.parseString(
+      f"""
+         |es.$namespace.host = $host
+         |es.$namespace.username = $username
+         |es.$namespace.password = $password
+         |""".stripMargin
     )
+
+    it("can build and accept ElasticConfigUsernamePassword") {
+      val elasticConfig = ElasticBuilder.buildElasticClientConfig(
+        usernamePasswordConfig,
+        namespace)
+
+      elasticConfig shouldBe ElasticConfigUsernamePassword(
+        host = host,
+        port = defaultPort,
+        protocol = defaultProtocol,
+        username = username,
+        password = password
+      )
+
+      noException shouldBe thrownBy(
+        ElasticBuilder.buildElasticClient(elasticConfig)
+      )
+    }
+
+    it("can build a client directly") {
+      noException shouldBe thrownBy(
+        ElasticBuilder.buildElasticClient(usernamePasswordConfig, namespace)
+      )
+    }
   }
 
-  it("builds a client when an API key is specified") {
-    val config = ConfigFactory.parseString(
-      """
-        |es.test.host = test.elastic
-        |es.test.apikey = test-key
-        |""".stripMargin
-    )
+  val apiKeyConfig = ConfigFactory.parseString(
+    f"""
+      |es.$namespace.host = $host
+      |es.$namespace.apikey = $apiKey
+      |""".stripMargin
+  )
 
-    noException shouldBe thrownBy(
-      ElasticBuilder.buildElasticClient(config, "test")
-    )
+  describe("when an API key is specified") {
+    it("can build and accept ElasticConfigApiKey") {
+      val elasticConfig =
+        ElasticBuilder.buildElasticClientConfig(apiKeyConfig, namespace)
+
+      elasticConfig shouldBe ElasticConfigApiKey(
+        host = host,
+        port = defaultPort,
+        protocol = defaultProtocol,
+        apiKey = apiKey
+      )
+
+      noException shouldBe thrownBy(
+        ElasticBuilder.buildElasticClient(elasticConfig)
+      )
+    }
+
+    it("builds a client directly") {
+      noException shouldBe thrownBy(
+        ElasticBuilder.buildElasticClient(apiKeyConfig, namespace)
+      )
+    }
+  }
+
+  describe("when an API key and username and password is specified") {
+    it("uses the API key") {
+      val config = ConfigFactory.parseString(
+        f"""
+           |es.$namespace.host = $host
+           |es.$namespace.username = $username
+           |es.$namespace.password = $password
+           |es.$namespace.apikey = $apiKey
+           |""".stripMargin
+      )
+
+      val elasticConfig =
+        ElasticBuilder.buildElasticClientConfig(config, namespace)
+
+      elasticConfig shouldBe ElasticConfigApiKey(
+        host = host,
+        port = defaultPort,
+        protocol = defaultProtocol,
+        apiKey = apiKey
+      )
+    }
   }
 
   it("errors if there is not enough config to build a client") {
     val config = ConfigFactory.parseString(
-      """
-        |es.test.host = test.elastic
-        |es.test.username = test-username
+      f"""
+        |es.$namespace.host = $host
+        |es.$namespace.username = $username
         |""".stripMargin
     )
 
     a[Throwable] shouldBe thrownBy(
-      ElasticBuilder.buildElasticClient(config, "test")
+      ElasticBuilder.buildElasticClient(config, namespace)
     )
   }
 }

fixtures/src/test/scala/weco/fixtures/LocalResources.scala

@@ -18,7 +18,8 @@ trait LocalResources {
       case Success(lines) => lines.mkString("\n")
 
       case Failure(_: NullPointerException) if name.startsWith("/") =>
-        throw new RuntimeException(s"Could not find resource `$name`; try removing the leading slash")
+        throw new RuntimeException(
+          s"Could not find resource `$name`; try removing the leading slash")
       case Failure(_: NullPointerException) =>
         throw new RuntimeException(s"Could not find resource `$name`")
 

messaging/src/test/scala/weco/messaging/sns/SNSMessageSenderTest.scala

@@ -34,7 +34,8 @@ class SNSMessageSenderTest
 
     val result = sender.send("hello world")(
       subject = "Sent from SNSMessageSenderTest",
-      destination = SNSConfig(topicArn = "arn:aws:sns:eu-west-1:012345678912:doesnotexist")
+      destination =
+        SNSConfig(topicArn = "arn:aws:sns:eu-west-1:012345678912:doesnotexist")
     )
 
     result shouldBe a[Failure[_]]

project/Common.scala

@@ -36,17 +36,17 @@ object Common {
       "-feature",
       "-language:postfixOps"
     ),
-    parallelExecution in Test := false,
+    Test / parallelExecution := false,
     publishMavenStyle := true,
     credentials += Credentials(Path.userHome / ".sbt" / "sonatype.credentials"),
     sonatypeCredentialHost := "central.sonatype.com",
     sonatypeRepository := "https://central.sonatype.com/service/local",
     licenses := Seq("MIT" -> url("https://github.com/wellcomecollection/scala-libs/blob/main/LICENSE")),
     publishTo := sonatypePublishToBundle.value,
-    publishArtifact in Test := true,
+    Test / publishArtifact := true,
     // Don't build scaladocs
     // https://www.scala-sbt.org/sbt-native-packager/formats/universal.html#skip-packagedoc-task-on-stage
-    mappings in (Compile, packageDoc) := Nil,
+    Compile /  packageDoc / mappings := Nil,
     version := projectVersion
   )
 

storage/docker-compose.yml

@@ -5,19 +5,9 @@ services:
     ports:
       - "45678:8000"
   s3:
-    image: "zenko/cloudserver:8.1.8"
+    image: "public.ecr.aws/localstack/localstack:4.0.0"
     environment:
-      - "S3BACKEND=mem"
+      - SERVICES=s3
+      - ALLOW_NONSTANDARD_REGIONS=1
     ports:
-      - "33333:8000"
-
-    # We've seen flakiness when this container doesn't start fast enough,
-    # and the first few tests to interact with S3 fail.  This uses
-    # docker-compose healthchecks to check the container is returning
-    # a 401 Unauthorized before continuing.
-    # See https://docs.docker.com/compose/compose-file/#/healthcheck
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:33333"]
-      interval: 2s
-      timeout: 10s
-      retries: 5
+      - "33333:4566"

storage/src/main/scala/weco/storage/providers/s3/S3Errors.scala

@@ -28,6 +28,10 @@ object S3Errors {
         if exc.getMessage.startsWith("The specified bucket is not valid") =>
       StoreReadError(exc)
 
+    case exc: S3Exception
+        if exc.getMessage.startsWith("The specified bucket does not exist") =>
+      StoreReadError(exc)
+
     case exc: SdkClientException
         if exc.getMessage.startsWith("Unable to execute HTTP request") =>
       new StoreReadError(exc) with RetryableError
@@ -54,6 +58,9 @@ object S3Errors {
     case exc: S3Exception
         if exc.getMessage.startsWith("Object key is too long") =>
       InvalidIdentifierFailure(exc)
+    case exc: S3Exception
+        if exc.getMessage.startsWith("Your key is too long") =>
+      InvalidIdentifierFailure(exc)
 
     case exc => StoreWriteError(exc)
   }