Batcher Lambda, infrastructure and build

builds/deploy_catalogue_pipeline.sh

@@ -71,6 +71,7 @@ ENV_TAG="env.$PIPELINE_DATE" "$ROOT/builds/update_ecr_image_tag.sh" \
 
 if [[ "$TASK" == "tag_images_and_deploy_services" ]]
 then
+  echo "Deploying ECS pipeline services to catalogue-$PIPELINE_DATE"
   CLUSTER="catalogue-$PIPELINE_DATE" "$ROOT/builds/deploy_ecs_services.sh" \
     id_minter \
     image_inferrer \
@@ -88,5 +89,9 @@ then
     transformer_miro \
     transformer_sierra \
     transformer_tei
+
+  echo "Deploying λ pipline services to catalogue-$PIPELINE_DATE"
+  "$ROOT/builds/deploy_lambda_services.sh" \
+    batcher
 fi
 

builds/deploy_lambda_image.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+PIPELINE_NAMESPACE="catalogue-$PIPELINE_DATE"
+REPOSITORY_URI="760097843905.dkr.ecr.eu-west-1.amazonaws.com"
+
+for SERVICE_NAME in "$@"
+do
+  IMAGE_URI="${REPOSITORY_URI}"/uk.ac.wellcome/"${SERVICE_NAME}":"env.${PIPELINE_DATE}"
+  FUNCTION_NAME="${PIPELINE_NAMESPACE}"-"${SERVICE_NAME}"
+
+  echo "Deploying ${IMAGE_URI} to ${FUNCTION_NAME}, @ $(date) ..."
+
+  echo "Current lambda configuration for ${FUNCTION_NAME}:"
+  aws lambda get-function-configuration \
+    --function-name "$FUNCTION_NAME" \
+    --no-cli-pager
+
+  echo "Updating lambda configuration ..."
+  echo "Using ${IMAGE_URI}:"
+  aws lambda update-function-code \
+    --function-name "$FUNCTION_NAME" \
+    --image-uri "${IMAGE_URI}" \
+    --no-cli-pager
+
+
+  echo "Updated lambda configuration, (waiting for update @ $(date)}):"
+  aws lambda wait function-updated \
+    --function-name "$FUNCTION_NAME" \
+    --no-cli-pager
+
+  echo "New lambda configuration complete (@ $(date)), config after change:"
+  aws lambda get-function-configuration \
+    --function-name "$FUNCTION_NAME" \
+    --no-cli-pager
+
+  echo "Done deploying ${FUNCTION_NAME} @ $(date)! 🚀"
+done

infrastructure/modules/scaling_service/main.tf

@@ -10,7 +10,7 @@ locals {
 }
 
 module "service" {
-  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/service?ref=v3.13.1"
+  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/service?ref=v4.1.0"
 
   task_definition_arn            = module.task_definition.arn
   service_name                   = local.service_name
@@ -34,7 +34,7 @@ module "service" {
 }
 
 module "autoscaling" {
-  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/autoscaling?ref=v3.13.1"
+  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/autoscaling?ref=v4.1.0"
 
   name = var.name
 
@@ -49,7 +49,7 @@ module "autoscaling" {
 }
 
 module "task_definition" {
-  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/task_definition?ref=v3.13.1"
+  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/task_definition?ref=v4.1.0"
 
   cpu    = var.cpu
   memory = var.memory
@@ -66,14 +66,14 @@ module "task_definition" {
 }
 
 module "log_router_container" {
-  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/firelens?ref=v3.13.1"
+  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/firelens?ref=v4.1.0"
   namespace = var.name
 
   use_privatelink_endpoint = true
 }
 
 module "log_router_permissions" {
-  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1"
+  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.1.0"
   secrets   = var.shared_logging_secrets
   role_name = module.task_definition.task_execution_role_name
 }

pipeline/relation_embedder/batcher/Dockerfile

@@ -1,9 +1,17 @@
-FROM public.ecr.aws/docker/library/eclipse-temurin:11-jre-alpine
+FROM public.ecr.aws/amazoncorretto/amazoncorretto:11 as base
 
 LABEL maintainer = "Wellcome Collection <[email protected]>"
 
-RUN apk add --no-cache bash
-
 ADD target/universal/stage /opt/docker
 
+WORKDIR /opt/docker
+
+FROM base as ecs
+
 ENTRYPOINT ["/opt/docker/bin/batcher"]
+
+FROM base
+
+ENTRYPOINT [ "/usr/bin/java", "-cp", "./lib/*", "com.amazonaws.services.lambda.runtime.api.client.AWSLambda" ]
+
+CMD ["weco.pipeline.batcher.LambdaMain::handleRequest"]

pipeline/relation_embedder/batcher/src/main/scala/weco/pipeline/batcher/LambdaMain.scala

@@ -0,0 +1,19 @@
+package weco.pipeline.batcher
+
+import com.amazonaws.services.lambda.runtime.{Context, RequestHandler}
+import grizzled.slf4j.Logging
+
+import java.util.{Map => JavaMap}
+
+object LambdaMain
+    extends RequestHandler[JavaMap[String, String], String]
+    with Logging {
+
+  override def handleRequest(
+    event: JavaMap[String, String],
+    context: Context
+  ): String = {
+    info(s"running batcher lambda, got event: $event")
+    "Done"
+  }
+}

pipeline/terraform/modules/fargate_service/main.tf

@@ -67,7 +67,7 @@ module "scaling_service" {
 }
 
 module "app_container" {
-  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v3.13.1"
+  source = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/container_definition?ref=v4.1.0"
 
   name  = local.name
   image = var.container_image
@@ -78,13 +78,16 @@ module "app_container" {
     var.env_vars,
   )
 
+  entrypoint = var.entrypoint
+  command    = var.command
+
   secrets = var.secret_env_vars
 
   log_configuration = module.scaling_service.log_configuration
 }
 
 module "app_permissions" {
-  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v3.13.1"
+  source    = "git::github.com/wellcomecollection/terraform-aws-ecs-service.git//modules/secrets?ref=v4.1.0"
   secrets   = var.secret_env_vars
   role_name = module.scaling_service.task_execution_role_name
 }

pipeline/terraform/modules/fargate_service/variables.tf

@@ -29,6 +29,16 @@ variable "queue_name" {
   default = null
 }
 
+variable "entrypoint" {
+  type    = list(string)
+  default = null
+}
+
+variable "command" {
+  default = null
+  type    = list(string)
+}
+
 variable "queue_visibility_timeout_seconds" {
   type    = number
   default = 30

pipeline/terraform/modules/pipeline_lambda/data.tf

@@ -0,0 +1,15 @@
+locals {
+  namespace = "catalogue-${var.pipeline_date}"
+
+  name      = "${local.namespace}-${var.service_name}"
+  image_tag = var.tag_override != "" ? var.tag_override : "env.${var.pipeline_date}"
+}
+
+data "aws_ecr_repository" "repository" {
+  name = var.ecr_repository_name
+}
+
+data "aws_ecr_image" "lambda_image" {
+  repository_name = data.aws_ecr_repository.repository.name
+  image_tag       = local.image_tag
+}

pipeline/terraform/modules/pipeline_lambda/main.tf

@@ -0,0 +1,14 @@
+module "pipeline_step" {
+  source = "github.com/wellcomecollection/terraform-aws-lambda.git?ref=v1.1.1"
+
+  name         = local.name
+  package_type = "Image"
+  image_uri    = "${data.aws_ecr_repository.repository.repository_url}@${data.aws_ecr_image.lambda_image.id}"
+  timeout      = var.timeout
+  memory_size  = var.memory_size
+  description  = var.description
+
+  environment = {
+    variables = var.environment_variables
+  }
+}

pipeline/terraform/modules/pipeline_lambda/variables.tf

@@ -0,0 +1,38 @@
+variable "tag_override" {
+  type    = string
+  default = ""
+}
+
+variable "ecr_repository_name" {
+  type = string
+}
+
+variable "service_name" {
+  type = string
+}
+
+variable "description" {
+  type    = string
+  default = ""
+}
+
+variable "pipeline_date" {
+  type = string
+}
+
+
+variable "environment_variables" {
+  type        = map(string)
+  description = "Arbitrary environment variables to give to the Lambda"
+  default     = {}
+}
+
+variable "timeout" {
+  default     = 600
+  description = "lambda function timeout"
+}
+
+variable "memory_size" {
+  default     = 1024
+  description = "lambda function memory size"
+}

pipeline/terraform/modules/stack/service_work_batcher.tf

@@ -30,12 +30,29 @@ module "batcher_output_topic" {
   role_names = [module.batcher.task_role_name]
 }
 
+module "batcher_lambda" {
+  source = "../pipeline_lambda"
+
+  pipeline_date = var.pipeline_date
+  service_name  = "batcher"
+  tag_override  = "dev"
+
+  ecr_repository_name = "uk.ac.wellcome/batcher"
+}
+
 module "batcher" {
   source = "../fargate_service"
 
   name            = "batcher"
   container_image = local.batcher_image
 
+  // Override entrypoint & command to dual use lambda container image
+  // This should be removed once we have a dedicated batcher_lambda image
+  entrypoint = [
+    "/opt/docker/bin/batcher"
+  ]
+  command = null
+
   topic_arns = [
     module.router_path_output_topic.arn,
     module.path_concatenator_output_topic.arn,

project/Dependencies.scala

@@ -87,6 +87,7 @@ object WellcomeDependencies {
 object ExternalDependencies {
   lazy val versions = new {
     val apacheCommons = "1.12.0"
+    val awsLambdaClient = "2.5.1"
     val circe = "0.14.1"
     val diffJson = "4.1.1"
     val fastparse = "2.3.3"
@@ -112,6 +113,10 @@ object ExternalDependencies {
     "org.apache.commons" % "commons-text" % versions.apacheCommons
   )
 
+  val awsLambdaClient: Seq[ModuleID] = Seq(
+    "com.amazonaws" % "aws-lambda-java-runtime-interface-client" % versions.awsLambdaClient
+  )
+
   val circeOpticsDependencies = Seq(
     "io.circe" %% "circe-optics" % versions.circe
   )
@@ -232,6 +237,7 @@ object CatalogueDependencies {
 
   val batcherDependencies: Seq[ModuleID] =
     ExternalDependencies.scalatestDependencies ++
+      ExternalDependencies.awsLambdaClient ++
       WellcomeDependencies.typesafeLibrary ++
       WellcomeDependencies.fixturesLibrary ++
       WellcomeDependencies.messagingTypesafeLibrary