From 4c581fd0d49e73b31c893d5871435d726a239d7c Mon Sep 17 00:00:00 2001 From: David Blatcher Date: Wed, 3 Jul 2024 10:40:09 +0100 Subject: [PATCH 1/4] upgrade aws version, replace deprecated ReceiveMessageRequest.withAttributeNames method --- build.sbt | 2 +- .../lib/aws/SimpleSqsMessageConsumer.scala | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index c9d8ee252b..7eb7d33a6d 100644 --- a/build.sbt +++ b/build.sbt @@ -54,7 +54,7 @@ Global / concurrentRestrictions := Seq( Tags.limitAll(12) ) -val awsSdkVersion = "1.12.470" +val awsSdkVersion = "1.12.755" val elastic4sVersion = "8.3.0" val okHttpVersion = "3.12.1" diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 3674600e06..d16d6d4419 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -11,13 +11,15 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { lazy val client: AmazonSQS = config.withAWSCredentials(AmazonSQSClientBuilder.standard()).build() - def getNextMessage(attributeNames: String*): Option[SQSMessage] = - client.receiveMessage( - new ReceiveMessageRequest(queueUrl) + def getNextMessage(attributeNames: String*): Option[SQSMessage] = { + val request = new ReceiveMessageRequest(queueUrl) .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .withAttributeNames(attributeNames: _*) - ).getMessages.asScala.headOption + + request.setMessageSystemAttributeNames(attributeNames.asJava) + + client.receiveMessage(request).getMessages.asScala.headOption + } def deleteMessage(message: SQSMessage): Unit = client.deleteMessage(new DeleteMessageRequest(queueUrl, message.getReceiptHandle)) From 1cf9a785adb12c9fe8f065fb651e27e9180681f4 Mon Sep 17 00:00:00 2001 From: David Blatcher Date: Wed, 3 Jul 2024 10:55:22 +0100 Subject: [PATCH 2/4] patch kinesis-logback-appender --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7eb7d33a6d..dd4de89fdc 100644 --- a/build.sbt +++ b/build.sbt @@ -88,7 +88,7 @@ lazy val commonLib = project("common-lib").settings( "com.gu" %% "thrift-serializer" % "5.0.2", "org.scalaz.stream" %% "scalaz-stream" % "0.8.6", "org.im4java" % "im4java" % "1.4.0", - "com.gu" % "kinesis-logback-appender" % "1.4.2", + "com.gu" % "kinesis-logback-appender" % "1.4.4", "net.logstash.logback" % "logstash-logback-encoder" % "5.0", "com.typesafe.play" %% "play-logback" % "2.8.20", // needed when running the scripts "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2", From 29c1e9e8615b066c64c5837ccff98c6be107a1b7 Mon Sep 17 00:00:00 2001 From: David Blatcher Date: Wed, 3 Jul 2024 14:46:19 +0100 Subject: [PATCH 3/4] attempt to debug the loader healthcheck and realise the ingestQueue does not exist anymore --- .../lib/aws/SimpleSqsMessageConsumer.scala | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index d16d6d4419..9870930dd7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -5,6 +5,7 @@ import com.amazonaws.services.sqs.AmazonSQS import com.amazonaws.services.sqs.AmazonSQSClientBuilder import com.amazonaws.services.sqs.model.{DeleteMessageRequest, ReceiveMessageRequest, Message => SQSMessage} +import scala.annotation.nowarn import scala.collection.JavaConverters._ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { @@ -12,11 +13,13 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { lazy val client: AmazonSQS = config.withAWSCredentials(AmazonSQSClientBuilder.standard()).build() def getNextMessage(attributeNames: String*): Option[SQSMessage] = { + + // TO DO - try the new methods when the IngestSqsQueue is back, getstatus works and healthcheck passes + @nowarn // withAttributeNames is deprecated, but still supported. val request = new ReceiveMessageRequest(queueUrl) .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - - request.setMessageSystemAttributeNames(attributeNames.asJava) + .withAttributeNames(attributeNames:_*) client.receiveMessage(request).getMessages.asScala.headOption } @@ -25,6 +28,15 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { client.deleteMessage(new DeleteMessageRequest(queueUrl, message.getReceiptHandle)) def getStatus: Map[String, String] = { + + println(queueUrl) + + println(client.getQueueAttributes(queueUrl, List( + "ApproximateNumberOfMessagesDelayed", + "ApproximateNumberOfMessages", + "ApproximateNumberOfMessagesNotVisible" + ).asJava)) + client.getQueueAttributes(queueUrl, List( "ApproximateNumberOfMessagesDelayed", "ApproximateNumberOfMessages", From f549e89b39649454460c9c88939e5d86eb79fae5 Mon Sep 17 00:00:00 2001 From: David Blatcher Date: Mon, 8 Jul 2024 11:56:46 +0100 Subject: [PATCH 4/4] try patching localstack version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 672e62bfd8..8aa31f7b10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: ports: - "9090:9000" localstack: - image: localstack/localstack:0.12.3 + image: localstack/localstack:0.12.20 platform: 'linux/x86_64' ports: - "4566:4566" # localstack's service proxy endpoint