From 598109a2faf888a606e65385fbebded1552da8a4 Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Fri, 24 May 2024 12:45:49 -0700 Subject: [PATCH 1/6] chore: add localstack to docker compose file --- docker-compose.dev.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 484f17ef..ff34c6c8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -20,6 +20,8 @@ services: - SESSION_SECRET=super_secret_session_key_change_me_in_production_please - CI= - NODE_ENV=development + - LOCALSTACK_HOSTNAME=localstack + - EDGE_PORT=4566 db: image: postgres:15-bookworm @@ -32,6 +34,22 @@ services: volumes: - postgres:/var/lib/postgresql/data + localstack: + container_name: "cpf-reporter-localstack-main" + image: localstack/localstack + ports: + - "4566:4566" # LocalStack Gateway + - "4510-4559:4510-4559" # external services port range + environment: + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + - AWS_DEFAULT_REGION=${AWS_REGION:-us-west-2} + - REPORTING_DATA_BUCKET_NAME=bucket + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" + - "./entrypoint/init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh" + # After starting with `docker compose -f ./docker-compose.dev.yml up`, # use the console to run commands in the container: # From cc7a4355ed8f58d4eff3e319a51c1f06a039a8c4 Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Fri, 24 May 2024 12:46:00 -0700 Subject: [PATCH 2/6] chore: Add script to test localstack --- scripts/tests3.ts | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 scripts/tests3.ts diff --git a/scripts/tests3.ts b/scripts/tests3.ts new file mode 100644 index 00000000..ee45adf5 --- /dev/null +++ b/scripts/tests3.ts @@ -0,0 +1,48 @@ +import { + GetObjectCommand, + CreateBucketCommand, + HeadObjectCommand, + HeadObjectCommandInput, + PutObjectCommand, + PutObjectCommandInput, + S3Client, +} from '@aws-sdk/client-s3' +import { getSignedUrl as awsGetSignedUrl } from '@aws-sdk/s3-request-presigner' +import type { Prisma } from '@prisma/client' +import { db, getPrismaClient } from 'api/src/lib/db' +import AWS from 'api/src/lib/aws' + +const REPORTING_DATA_BUCKET_NAME = `${process.env.REPORTING_DATA_BUCKET_NAME}` + +export default async () => { + try { + await getPrismaClient() + + const s3 = AWS.getS3Client() + const key = "12345" + const baseParams: PutObjectCommandInput = { + Bucket: REPORTING_DATA_BUCKET_NAME, + Key: key, + ContentType: 'application/vnd.ms-excel.sheet.macroenabled.12', + ServerSideEncryption: 'AES256', + } + const url = await awsGetSignedUrl(s3, new PutObjectCommand(baseParams), { + expiresIn: 60, + }) + console.log("url", url) + + const uploadParams: HeadObjectCommandInput = { + Bucket: REPORTING_DATA_BUCKET_NAME, + Key: key, + } + await s3.send(new CreateBucketCommand({ + Bucket: REPORTING_DATA_BUCKET_NAME, + })) + await s3.send(new HeadObjectCommand(uploadParams)) + + + } catch (error) { + console.warn('Failed to test s3.') + console.error(error) + } +} From 7e44825a51ccf661fa59a752a7b3efa07b9fdb5c Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Tue, 28 May 2024 10:42:55 -0700 Subject: [PATCH 3/6] chore: add localstack hostname to more containers --- docker-compose.dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ff34c6c8..cd62d828 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -72,6 +72,7 @@ services: environment: - DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood - TEST_DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood_test + - LOCALSTACK_HOSTNAME=localstack depends_on: - db @@ -90,6 +91,7 @@ services: environment: - DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood - TEST_DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood_test + - LOCALSTACK_HOSTNAME=localstack depends_on: - db From bf908a073695871d50e3b752238b90952b7fda5a Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Tue, 28 May 2024 10:43:04 -0700 Subject: [PATCH 4/6] chore: make example script more readable --- scripts/tests3.ts | 73 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/scripts/tests3.ts b/scripts/tests3.ts index ee45adf5..3c25903f 100644 --- a/scripts/tests3.ts +++ b/scripts/tests3.ts @@ -1,48 +1,83 @@ import { - GetObjectCommand, CreateBucketCommand, + GetObjectCommand, + GetObjectCommandInput, HeadObjectCommand, HeadObjectCommandInput, PutObjectCommand, PutObjectCommandInput, - S3Client, + DeleteObjectCommand, + DeleteObjectCommandInput, + DeleteBucketCommand, + DeleteBucketCommandInput, } from '@aws-sdk/client-s3' import { getSignedUrl as awsGetSignedUrl } from '@aws-sdk/s3-request-presigner' -import type { Prisma } from '@prisma/client' -import { db, getPrismaClient } from 'api/src/lib/db' +import { getPrismaClient } from 'api/src/lib/db' import AWS from 'api/src/lib/aws' const REPORTING_DATA_BUCKET_NAME = `${process.env.REPORTING_DATA_BUCKET_NAME}` export default async () => { - try { - await getPrismaClient() + const s3 = AWS.getS3Client() + await getPrismaClient() + const key = "12345" - const s3 = AWS.getS3Client() - const key = "12345" - const baseParams: PutObjectCommandInput = { + console.log("Create bucket") + try { + await s3.send(new CreateBucketCommand({ Bucket: REPORTING_DATA_BUCKET_NAME, - Key: key, - ContentType: 'application/vnd.ms-excel.sheet.macroenabled.12', - ServerSideEncryption: 'AES256', + })) + console.log("-- Bucket created") + } catch (error) { + if (error.Code == "BucketAlreadyOwnedByYou") { + console.log("-- Bucket already exists") + } else { + console.log(error) } - const url = await awsGetSignedUrl(s3, new PutObjectCommand(baseParams), { + } + + const baseParams: GetObjectCommandInput = { + Bucket: REPORTING_DATA_BUCKET_NAME, + Key: key, + } + try { + console.log("Create signed URL") + const url = await awsGetSignedUrl(s3, new GetObjectCommand(baseParams), { expiresIn: 60, }) - console.log("url", url) + console.log("-- signed url", url) - const uploadParams: HeadObjectCommandInput = { + console.log("Upload file") + const uploadParams: PutObjectCommandInput = { + Body: "tests3.ts", // Upload this file as a test Bucket: REPORTING_DATA_BUCKET_NAME, Key: key, } - await s3.send(new CreateBucketCommand({ + await s3.send(new PutObjectCommand(uploadParams)) + + console.log("-- Look at uploaded file") + const headParams: HeadObjectCommandInput = { Bucket: REPORTING_DATA_BUCKET_NAME, - })) - await s3.send(new HeadObjectCommand(uploadParams)) - + Key: key, + } + const response = await s3.send(new HeadObjectCommand(headParams)) + console.log("--", response) + console.log("Delete file") + const deleteParams: DeleteObjectCommandInput = { + Bucket: REPORTING_DATA_BUCKET_NAME, + Key: key, + } + await s3.send(new DeleteObjectCommand(headParams)) + console.log("-- file deleted") } catch (error) { console.warn('Failed to test s3.') console.error(error) + } finally { + console.log("Remove bucket") + await s3.send(new DeleteBucketCommand({ + Bucket: REPORTING_DATA_BUCKET_NAME, + } as DeleteBucketCommandInput)) + console.log("-- Bucket removed") } } From 8dd4833969bb2fc81c215e1ae4ef55ce9cb3e61c Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Sat, 1 Jun 2024 15:53:54 -0700 Subject: [PATCH 5/6] chore: update script to use different bucket --- scripts/tests3.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests3.ts b/scripts/tests3.ts index 3c25903f..9d2ee51a 100644 --- a/scripts/tests3.ts +++ b/scripts/tests3.ts @@ -1,3 +1,4 @@ +import { v4 as uuidv4 } from "uuid" import { CreateBucketCommand, GetObjectCommand, @@ -15,7 +16,7 @@ import { getSignedUrl as awsGetSignedUrl } from '@aws-sdk/s3-request-presigner' import { getPrismaClient } from 'api/src/lib/db' import AWS from 'api/src/lib/aws' -const REPORTING_DATA_BUCKET_NAME = `${process.env.REPORTING_DATA_BUCKET_NAME}` +const REPORTING_DATA_BUCKET_NAME = `test-${process.env.REPORTING_DATA_BUCKET_NAME}-${uuidv4()}` export default async () => { const s3 = AWS.getS3Client() @@ -68,7 +69,7 @@ export default async () => { Bucket: REPORTING_DATA_BUCKET_NAME, Key: key, } - await s3.send(new DeleteObjectCommand(headParams)) + await s3.send(new DeleteObjectCommand(deleteParams)) console.log("-- file deleted") } catch (error) { console.warn('Failed to test s3.') From da5ba65cb174fbce9dad6c7a087ab84f8b202f85 Mon Sep 17 00:00:00 2001 From: Victor Shia <401047+vshia@users.noreply.github.com> Date: Tue, 4 Jun 2024 09:15:15 -0700 Subject: [PATCH 6/6] Update docker-compose.dev.yml --- docker-compose.dev.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cd62d828..1a02fef6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -41,10 +41,13 @@ services: - "4566:4566" # LocalStack Gateway - "4510-4559:4510-4559" # external services port range environment: + - SERVICES=s3, iam, lambda, cloudwatch - DEBUG=${DEBUG-} - DOCKER_HOST=unix:///var/run/docker.sock - AWS_DEFAULT_REGION=${AWS_REGION:-us-west-2} - REPORTING_DATA_BUCKET_NAME=bucket + - AWS_ACCESS_KEY_ID="test" + - AWS_SECRET_ACCESS_KEY="test" volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock"