feat(attachments infrastructure): Bucket, file scanning, and presigne…

…d urls. (#176)

* uploads serfvice and presigned url generation

* add missing export

serverless-compose.yml

@@ -9,6 +9,8 @@ services:
     path: src/services/data
     params:
       ECSFailureTopicArn: ${alerts.ECSFailureTopicArn}
+  uploads:
+    path: src/services/uploads
   ui-infra:
     path: src/services/ui-infra
   api:
@@ -17,6 +19,10 @@ services:
       ECSFailureTopicArn: ${alerts.ECSFailureTopicArn}
       osDomainArn: ${data.OpenSearchDomainArn}
       osDomain: ${data.OpenSearchDomainEndpoint}
+      topicName: ${data.TopicName}
+      attachmentsBucketName: ${uploads.AttachmentsBucketName}
+      attachmentsBucketRegion: ${uploads.AttachmentsBucketRegion}
+      attachmentsBucketArn: ${uploads.AttachmentsBucketArn}
   auth:
     path: src/services/auth
     params:

src/services/api/handlers/getAttachmentUrl.ts

@@ -69,7 +69,7 @@ export const handler = async (event: APIGatewayEvent) => {
     }
 
     // Now we can generate the presigned url
-    const url = await generatePresignedS3Url(body.bucket, body.key, 60);
+    const url = await generateLegacyPresignedS3Url(body.bucket, body.key, 60);
 
     return response<unknown>({
       statusCode: 200,
@@ -84,32 +84,37 @@ export const handler = async (event: APIGatewayEvent) => {
   }
 };
 
-async function generatePresignedS3Url(bucket, key, expirationInSeconds) {
-  // Create an S3 client
-  const roleToAssumeArn = process.env.onemacLegacyS3AccessRoleArn;
-
-  // Create an STS client to make the AssumeRole API call
-  const stsClient = new STSClient({});
-
-  // Assume the role
-  const assumedRoleResponse = await stsClient.send(
-    new AssumeRoleCommand({
-      RoleArn: roleToAssumeArn,
-      RoleSessionName: "AssumedRoleSession",
-    })
-  );
-
-  // Extract the assumed role credentials
-  const assumedCredentials = assumedRoleResponse.Credentials;
-
-  // Create S3 client using the assumed role's credentials
-  const assumedS3Client = new S3Client({
-    credentials: {
-      accessKeyId: assumedCredentials.AccessKeyId,
-      secretAccessKey: assumedCredentials.SecretAccessKey,
-      sessionToken: assumedCredentials.SessionToken,
-    },
-  });
+async function getClient(bucket) {
+  if (bucket.startsWith("uploads")) {
+    const stsClient = new STSClient({});
+
+    // Assume the role
+    const assumedRoleResponse = await stsClient.send(
+      new AssumeRoleCommand({
+        RoleArn: process.env.onemacLegacyS3AccessRoleArn,
+        RoleSessionName: "AssumedRoleSession",
+      })
+    );
+
+    // Extract the assumed role credentials
+    const assumedCredentials = assumedRoleResponse.Credentials;
+
+    // Create S3 client using the assumed role's credentials
+    return new S3Client({
+      credentials: {
+        accessKeyId: assumedCredentials.AccessKeyId,
+        secretAccessKey: assumedCredentials.SecretAccessKey,
+        sessionToken: assumedCredentials.SessionToken,
+      },
+    });
+  } else {
+    return new S3Client({});
+  }
+}
+
+async function generateLegacyPresignedS3Url(bucket, key, expirationInSeconds) {
+  // Get an S3 client
+  const client = await getClient(bucket);
 
   // Create a command to get the object (you can adjust this according to your use case)
   const getObjectCommand = new GetObjectCommand({
@@ -118,7 +123,7 @@ async function generatePresignedS3Url(bucket, key, expirationInSeconds) {
   });
 
   // Generate a presigned URL
-  const presignedUrl = await getSignedUrl(assumedS3Client, getObjectCommand, {
+  const presignedUrl = await getSignedUrl(client, getObjectCommand, {
     expiresIn: expirationInSeconds,
   });
 

src/services/api/handlers/getUploadUrl.ts

@@ -0,0 +1,52 @@
+import { response } from "../libs/handler";
+import { APIGatewayEvent } from "aws-lambda";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { v4 as uuidv4 } from "uuid";
+
+checkEnvVariables(["attachmentsBucketName", "attachmentsBucketRegion"]);
+
+const s3 = new S3Client({
+  region: process.env.attachmentsBucketRegion,
+});
+
+export const handler = async (event: APIGatewayEvent) => {
+  try {
+    const body = JSON.parse(event.body);
+    const bucket = process.env.attachmentsBucketName;
+    const key = uuidv4();
+    const url = await getSignedUrl(
+      s3,
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: key,
+      }),
+      {
+        expiresIn: 60,
+      }
+    );
+
+    return response<unknown>({
+      statusCode: 200,
+      body: { url, bucket, key },
+    });
+  } catch (error) {
+    console.error({ error });
+    return response({
+      statusCode: 500,
+      body: { message: "Internal server error" },
+    });
+  }
+};
+
+function checkEnvVariables(requiredEnvVariables) {
+  const missingVariables = requiredEnvVariables.filter(
+    (envVar) => !process.env[envVar]
+  );
+
+  if (missingVariables.length > 0) {
+    throw new Error(
+      `Missing required environment variables: ${missingVariables.join(", ")}`
+    );
+  }
+}

src/services/api/serverless.yml

@@ -39,6 +39,14 @@ provider:
             - sts:AssumeRole
           Resource:
             - ${self:custom.onemacLegacyS3AccessRoleArn}
+        - Effect: Allow
+          Action:
+            - s3:PutObject
+            - s3:PutObjectTagging
+            - s3:GetObject
+            - s3:GetObjectTagging
+          Resource:
+            - ${param:attachmentsBucketArn}/*
 
 custom:
   project: ${env:PROJECT}
@@ -52,6 +60,7 @@ custom:
   vpc: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/vpc, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/vpc}
   onemacLegacyS3AccessRoleArn: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/onemacLegacyS3AccessRoleArn, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/onemacLegacyS3AccessRoleArn}
   dbInfo: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/seatool/dbInfo, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/seatool/dbInfo}
+  brokerString: ${ssm:/aws/reference/secretsmanager/${self:custom.project}/${sls:stage}/brokerString, ssm:/aws/reference/secretsmanager/${self:custom.project}/default/brokerString}
   scriptable:
     hooks:
       package:compileEvents: ./handlers/repack.js
@@ -115,6 +124,17 @@ functions:
       subnetIds: >-
         ${self:custom.vpc.privateSubnets}
     provisionedConcurrency: ${param:getAttachmentUrlProvisionedConcurrency}
+  getUploadUrl:
+    handler: handlers/getUploadUrl.handler
+    environment:
+      attachmentsBucketName: ${param:attachmentsBucketName}
+      attachmentsBucketRegion: ${param:attachmentsBucketRegion}
+    events:
+      - http:
+          path: /getUploadUrl
+          method: post
+          cors: true
+          authorizer: aws_iam
   item:
     handler: handlers/item.handler
     maximumRetryAttempts: 0
@@ -142,6 +162,8 @@ functions:
       dbPort: ${self:custom.dbInfo.port}
       dbUser: ${self:custom.dbInfo.user}
       dbPassword: ${self:custom.dbInfo.password}
+      topicName: ${param:topicName}
+      brokerString: ${self:custom.brokerString}
     events:
       - http:
           path: /submit

src/services/data/serverless.yml

@@ -78,7 +78,6 @@ custom:
         then
           aws lambda invoke --region ${self:provider.region} --function-name ${self:service}-${sls:stage}-bootstrapUsers --invocation-type RequestResponse /dev/null
         fi
-
 stepFunctions:
   stateMachines:
     reindex:
@@ -681,3 +680,5 @@ resources:
       Value: !Sub https://${OpenSearch.DomainEndpoint}
     OpenSearchDashboardEndpoint:
       Value: !Sub https://${OpenSearch.DomainEndpoint}/_dashboards
+    TopicName:
+      Value: ${param:topicNamespace}aws.onemac.migration.cdc

src/services/uploads/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "uploads",
+  "description": "",
+  "private": true,
+  "version": "0.0.0",
+  "main": "index.js",
+  "scripts": {},
+  "author": "",
+  "license": "CC0-1.0",
+  "devDependencies": {}
+}