From 0d43abcf03b2bd6e68508737f0b77bbd17f765a7 Mon Sep 17 00:00:00 2001 From: David Pepper Date: Tue, 5 Sep 2023 15:23:53 +0100 Subject: [PATCH 1/5] add aws-sdk to package.json --- handlers/failed-national-deliveries-processor/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handlers/failed-national-deliveries-processor/package.json b/handlers/failed-national-deliveries-processor/package.json index 94b3a7c6c7..7d1a9135fd 100644 --- a/handlers/failed-national-deliveries-processor/package.json +++ b/handlers/failed-national-deliveries-processor/package.json @@ -7,7 +7,8 @@ "build": "esbuild --bundle --platform=node --target=node18 --outfile=dist/index.js src/index.ts" }, "devDependencies": { - "esbuild": "^0.19.2" + "esbuild": "^0.19.2", + "aws-sdk": "^2.1450.0" }, "dependencies": { "csv-parse": "^5.4.1", From 84f78bc26a0de35167be8fc9fa56b5a9665e26b1 Mon Sep 17 00:00:00 2001 From: David Pepper Date: Wed, 6 Sep 2023 14:08:04 +0100 Subject: [PATCH 2/5] add s3 and credentials files --- handlers/common/src/aws/credentials.ts | 32 ++++++++++++++++++++++++++ handlers/common/src/aws/s3.ts | 29 +++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 handlers/common/src/aws/credentials.ts create mode 100644 handlers/common/src/aws/s3.ts diff --git a/handlers/common/src/aws/credentials.ts b/handlers/common/src/aws/credentials.ts new file mode 100644 index 0000000000..b8bcefc562 --- /dev/null +++ b/handlers/common/src/aws/credentials.ts @@ -0,0 +1,32 @@ +import { STS } from "aws-sdk"; +import { + CredentialProviderChain, + Credentials, + EnvironmentCredentials, + SharedIniFileCredentials, +} from 'aws-sdk/lib/core'; + +export function credentialProvider(profile: string): CredentialProviderChain { + return new CredentialProviderChain([ + (): Credentials => new SharedIniFileCredentials({profile}), + (): Credentials => new EnvironmentCredentials('AWS'), + ]); +} + +export function assumeRole(roleToAssume: string): Promise { + const sts = new STS({apiVersion: '2011-06-15'}); + return sts.assumeRole({ + RoleArn: roleToAssume, + RoleSessionName: 'tracker', + }).promise().then((data) => { + if (data.Credentials) { + return new Credentials({ + accessKeyId: data.Credentials.AccessKeyId, + secretAccessKey: data.Credentials.SecretAccessKey, + sessionToken: data.Credentials.SessionToken + }) + } else { + return Promise.reject("Could not get credentials from AWS") + } + }) +} \ No newline at end of file diff --git a/handlers/common/src/aws/s3.ts b/handlers/common/src/aws/s3.ts new file mode 100644 index 0000000000..c545346348 --- /dev/null +++ b/handlers/common/src/aws/s3.ts @@ -0,0 +1,29 @@ +import { S3 } from 'aws-sdk'; +import type { GetObjectRequest, PutObjectRequest } from 'aws-sdk/clients/s3'; +import { credentialProvider } from './credentials'; + +export class S3Ops { + private readonly s3: S3; + + constructor(region: string) { + console.log('In constructor...'); + console.log('region:', region); + + this.s3 = new S3({ credentialProvider: credentialProvider('membership'), region }); + + } + + async getObject(params: GetObjectRequest): Promise { + console.log('In getObject...'); + console.log('params:',params); + + const response = await this.s3.getObject(params).promise(); + const body = response.Body?.toString(); + + if (!body) { + throw new Error(`s3://${params.Bucket}/${params.Key} is empty`); + } + + return body; + } +} \ No newline at end of file From e17f0ec1e3b1d82ef20da299c6d3758739137dd7 Mon Sep 17 00:00:00 2001 From: David Pepper Date: Wed, 6 Sep 2023 14:09:22 +0100 Subject: [PATCH 3/5] Revert "add s3 and credentials files" This reverts commit 84f78bc26a0de35167be8fc9fa56b5a9665e26b1. --- handlers/common/src/aws/credentials.ts | 32 -------------------------- handlers/common/src/aws/s3.ts | 29 ----------------------- 2 files changed, 61 deletions(-) delete mode 100644 handlers/common/src/aws/credentials.ts delete mode 100644 handlers/common/src/aws/s3.ts diff --git a/handlers/common/src/aws/credentials.ts b/handlers/common/src/aws/credentials.ts deleted file mode 100644 index b8bcefc562..0000000000 --- a/handlers/common/src/aws/credentials.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { STS } from "aws-sdk"; -import { - CredentialProviderChain, - Credentials, - EnvironmentCredentials, - SharedIniFileCredentials, -} from 'aws-sdk/lib/core'; - -export function credentialProvider(profile: string): CredentialProviderChain { - return new CredentialProviderChain([ - (): Credentials => new SharedIniFileCredentials({profile}), - (): Credentials => new EnvironmentCredentials('AWS'), - ]); -} - -export function assumeRole(roleToAssume: string): Promise { - const sts = new STS({apiVersion: '2011-06-15'}); - return sts.assumeRole({ - RoleArn: roleToAssume, - RoleSessionName: 'tracker', - }).promise().then((data) => { - if (data.Credentials) { - return new Credentials({ - accessKeyId: data.Credentials.AccessKeyId, - secretAccessKey: data.Credentials.SecretAccessKey, - sessionToken: data.Credentials.SessionToken - }) - } else { - return Promise.reject("Could not get credentials from AWS") - } - }) -} \ No newline at end of file diff --git a/handlers/common/src/aws/s3.ts b/handlers/common/src/aws/s3.ts deleted file mode 100644 index c545346348..0000000000 --- a/handlers/common/src/aws/s3.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { S3 } from 'aws-sdk'; -import type { GetObjectRequest, PutObjectRequest } from 'aws-sdk/clients/s3'; -import { credentialProvider } from './credentials'; - -export class S3Ops { - private readonly s3: S3; - - constructor(region: string) { - console.log('In constructor...'); - console.log('region:', region); - - this.s3 = new S3({ credentialProvider: credentialProvider('membership'), region }); - - } - - async getObject(params: GetObjectRequest): Promise { - console.log('In getObject...'); - console.log('params:',params); - - const response = await this.s3.getObject(params).promise(); - const body = response.Body?.toString(); - - if (!body) { - throw new Error(`s3://${params.Bucket}/${params.Key} is empty`); - } - - return body; - } -} \ No newline at end of file From 4961be2399672473f1102950c499d3fc22215e2a Mon Sep 17 00:00:00 2001 From: David Pepper Date: Wed, 6 Sep 2023 14:09:52 +0100 Subject: [PATCH 4/5] add credentials and s3 classes --- .../src/credentials.ts | 32 ++++++++++ .../src/s3.ts | 61 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 handlers/failed-national-deliveries-processor/src/credentials.ts create mode 100644 handlers/failed-national-deliveries-processor/src/s3.ts diff --git a/handlers/failed-national-deliveries-processor/src/credentials.ts b/handlers/failed-national-deliveries-processor/src/credentials.ts new file mode 100644 index 0000000000..b8bcefc562 --- /dev/null +++ b/handlers/failed-national-deliveries-processor/src/credentials.ts @@ -0,0 +1,32 @@ +import { STS } from "aws-sdk"; +import { + CredentialProviderChain, + Credentials, + EnvironmentCredentials, + SharedIniFileCredentials, +} from 'aws-sdk/lib/core'; + +export function credentialProvider(profile: string): CredentialProviderChain { + return new CredentialProviderChain([ + (): Credentials => new SharedIniFileCredentials({profile}), + (): Credentials => new EnvironmentCredentials('AWS'), + ]); +} + +export function assumeRole(roleToAssume: string): Promise { + const sts = new STS({apiVersion: '2011-06-15'}); + return sts.assumeRole({ + RoleArn: roleToAssume, + RoleSessionName: 'tracker', + }).promise().then((data) => { + if (data.Credentials) { + return new Credentials({ + accessKeyId: data.Credentials.AccessKeyId, + secretAccessKey: data.Credentials.SecretAccessKey, + sessionToken: data.Credentials.SessionToken + }) + } else { + return Promise.reject("Could not get credentials from AWS") + } + }) +} \ No newline at end of file diff --git a/handlers/failed-national-deliveries-processor/src/s3.ts b/handlers/failed-national-deliveries-processor/src/s3.ts new file mode 100644 index 0000000000..5703b255ae --- /dev/null +++ b/handlers/failed-national-deliveries-processor/src/s3.ts @@ -0,0 +1,61 @@ +import { S3 } from 'aws-sdk'; +import type { GetObjectRequest, PutObjectRequest } from 'aws-sdk/clients/s3'; +import { credentialProvider } from 'common/aws/credentials'; + +export class S3Ops { + private readonly s3: S3; + + constructor(region: string) { + // this.s3 = new S3({ credentialProvider: credentialProvider('deployTools'), region }); + console.log('HELLO!'); + } + + + + async greeting(){ + console.log('HELLO!'); + } + + async getObject(params: GetObjectRequest): Promise { + const response = await this.s3.getObject(params).promise(); + const body = response.Body?.toString(); + + if (!body) { + throw new Error(`s3://${params.Bucket}/${params.Key} is empty`); + } + + return body; + } + + async putObject( + bucketName: string, + key: string, + body: unknown, + ): Promise { + if (!shouldOutput) { + console.info( + `Skipping output. Would have put the following data into the following s3 bucket: ${key}`, + ); + console.info(JSON.stringify(body, null, 2)); + return Promise.resolve(); + } + + try { + const params: PutObjectRequest = { + Bucket: bucketName, + Key: key, + Body: JSON.stringify(body), + ContentType: 'application/json; charset=utf-8', + ACL: 'private', + }; + + await this.s3.upload(params).promise(); + } catch (e) { + if (e instanceof Error) { + console.error(`Error uploading item to s3: ${e.message}`); + } else { + console.error(e); + } + } + } +} \ No newline at end of file From df2a1e946fb250ac1e61282b85d05b2dca835c59 Mon Sep 17 00:00:00 2001 From: David Pepper Date: Wed, 6 Sep 2023 14:10:06 +0100 Subject: [PATCH 5/5] output s3 client for debugging --- .../src/index.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/handlers/failed-national-deliveries-processor/src/index.ts b/handlers/failed-national-deliveries-processor/src/index.ts index bc51c32b5a..d987d64bdd 100644 --- a/handlers/failed-national-deliveries-processor/src/index.ts +++ b/handlers/failed-national-deliveries-processor/src/index.ts @@ -1,17 +1,30 @@ import {Request, FileRow} from "../src/types.ts"; +import {S3Ops} from "../src/s3.ts"; import {getFileContents, getFailedDeliveryRowsFromFileContents} from "./FileIO.js"; import {generateRequestsFromFailedDeliveryRows} from "./RequestBuilder.js"; +// import { S3Ops } from 'common/aws/s3'; + export const main = async (): Promise => { - const allFileRows = await getFileContents(); - const failedDeliveryRows : FileRow[] = getFailedDeliveryRowsFromFileContents(allFileRows); + // const s3Client = new S3Ops(); + + const s3Client = new S3Ops('AWS_REGION'); + console.log('s3Client:',s3Client); + // await s3Client.putObject( + // config.bucketName, + // 'snyk/allrepos.json', + // allReposAndSnykProjects, + // ); + + // const allFileRows = await getFileContents(); + // const failedDeliveryRows : FileRow[] = getFailedDeliveryRowsFromFileContents(allFileRows); - const requests : Request[] = generateRequestsFromFailedDeliveryRows(failedDeliveryRows); + // const requests : Request[] = generateRequestsFromFailedDeliveryRows(failedDeliveryRows); - requests.forEach( - request => console.log('request: '+ JSON.stringify(request)) - ); + // requests.forEach( + // request => console.log('request: '+ JSON.stringify(request)) + // ); return Promise.resolve(''); };