Skip to content

Commit

Permalink
Merge pull request #2651 from guardian/discount-expiry-notifier-opera…
Browse files Browse the repository at this point in the history
…tions-status-to-s3

Discount expiry notifier - save operations status to s3
  • Loading branch information
david-pepper authored Jan 28, 2025
2 parents 483787d + 379e9ba commit 473c357
Show file tree
Hide file tree
Showing 8 changed files with 1,144 additions and 376 deletions.
34 changes: 18 additions & 16 deletions cdk/lib/__snapshots__/discount-expiry-notifier.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "CODE",
"Stage": "CODE",
},
},
"FunctionName": "discount-expiry-notifier-build-email-payload-CODE",
Expand Down Expand Up @@ -401,7 +400,7 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"Arn",
],
},
"","Payload.$":"$"}},"Is Subscription Active?":{"Type":"Choice","Choices":[{"Variable":"$.isActive","BooleanEquals":true,"Next":"Build email payload"}],"Default":"Skip Processing"},"Skip Processing":{"Type":"Pass","ResultPath":null,"End":true},"Build email payload":{"Next":"Initiate email send","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
"","Payload.$":"$"}},"Is Subscription Active?":{"Type":"Choice","Choices":[{"Variable":"$.status","StringEquals":"Active","Next":"Build email payload"}],"Default":"Skip Processing"},"Skip Processing":{"Type":"Pass","ResultPath":null,"End":true},"Build email payload":{"Next":"Initiate email send","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
{
"Ref": "AWS::Partition",
},
Expand All @@ -423,7 +422,7 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"Arn",
],
},
"","Payload.$":"$"}}}},"ItemsPath":"$.discountsToProcess","MaxConcurrency":10},"Save results":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
"","Payload.$":"$"}}}},"ItemsPath":"$.expiringDiscountsToProcess","MaxConcurrency":10},"Save results":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
{
"Ref": "AWS::Partition",
},
Expand Down Expand Up @@ -676,7 +675,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "CODE",
"Stage": "CODE",
},
},
"FunctionName": "discount-expiry-notifier-get-subs-with-expiring-discounts-CODE",
Expand Down Expand Up @@ -735,7 +733,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "CODE",
"Stage": "CODE",
},
},
"FunctionName": "discount-expiry-notifier-initiate-email-send-CODE",
Expand Down Expand Up @@ -1101,9 +1098,11 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"Environment": {
"Variables": {
"APP": "discount-expiry-notifier",
"S3_BUCKET": {
"Ref": "Bucket83908E77",
},
"STACK": "membership",
"STAGE": "CODE",
"Stage": "CODE",
},
},
"FunctionName": "discount-expiry-notifier-save-results-CODE",
Expand Down Expand Up @@ -1200,7 +1199,10 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"PolicyDocument": {
"Statement": [
{
"Action": "s3:PutObject",
"Action": [
"s3:GetObject",
"s3:PutObject",
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
Expand Down Expand Up @@ -1333,7 +1335,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 1`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "CODE",
"Stage": "CODE",
},
},
"FunctionName": "discount-expiry-notifier-sub-is-active-CODE",
Expand Down Expand Up @@ -1725,7 +1726,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "PROD",
"Stage": "PROD",
},
},
"FunctionName": "discount-expiry-notifier-build-email-payload-PROD",
Expand Down Expand Up @@ -1950,7 +1950,7 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"Arn",
],
},
"","Payload.$":"$"}},"Is Subscription Active?":{"Type":"Choice","Choices":[{"Variable":"$.isActive","BooleanEquals":true,"Next":"Build email payload"}],"Default":"Skip Processing"},"Skip Processing":{"Type":"Pass","ResultPath":null,"End":true},"Build email payload":{"Next":"Initiate email send","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
"","Payload.$":"$"}},"Is Subscription Active?":{"Type":"Choice","Choices":[{"Variable":"$.status","StringEquals":"Active","Next":"Build email payload"}],"Default":"Skip Processing"},"Skip Processing":{"Type":"Pass","ResultPath":null,"End":true},"Build email payload":{"Next":"Initiate email send","Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
{
"Ref": "AWS::Partition",
},
Expand All @@ -1972,7 +1972,7 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"Arn",
],
},
"","Payload.$":"$"}}}},"ItemsPath":"$.discountsToProcess","MaxConcurrency":10},"Save results":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
"","Payload.$":"$"}}}},"ItemsPath":"$.expiringDiscountsToProcess","MaxConcurrency":10},"Save results":{"End":true,"Retry":[{"ErrorEquals":["Lambda.ClientExecutionTimeoutException","Lambda.ServiceException","Lambda.AWSLambdaException","Lambda.SdkClientException"],"IntervalSeconds":2,"MaxAttempts":6,"BackoffRate":2}],"Type":"Task","OutputPath":"$.Payload","Resource":"arn:",
{
"Ref": "AWS::Partition",
},
Expand Down Expand Up @@ -2225,7 +2225,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "PROD",
"Stage": "PROD",
},
},
"FunctionName": "discount-expiry-notifier-get-subs-with-expiring-discounts-PROD",
Expand Down Expand Up @@ -2284,7 +2283,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "PROD",
"Stage": "PROD",
},
},
"FunctionName": "discount-expiry-notifier-initiate-email-send-PROD",
Expand Down Expand Up @@ -2650,9 +2648,11 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"Environment": {
"Variables": {
"APP": "discount-expiry-notifier",
"S3_BUCKET": {
"Ref": "Bucket83908E77",
},
"STACK": "membership",
"STAGE": "PROD",
"Stage": "PROD",
},
},
"FunctionName": "discount-expiry-notifier-save-results-PROD",
Expand Down Expand Up @@ -2749,7 +2749,10 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"PolicyDocument": {
"Statement": [
{
"Action": "s3:PutObject",
"Action": [
"s3:GetObject",
"s3:PutObject",
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
Expand Down Expand Up @@ -2882,7 +2885,6 @@ exports[`The discount-expiry-notifier stack matches the snapshot 2`] = `
"APP": "discount-expiry-notifier",
"STACK": "membership",
"STAGE": "PROD",
"Stage": "PROD",
},
},
"FunctionName": "discount-expiry-notifier-sub-is-active-PROD",
Expand Down
20 changes: 4 additions & 16 deletions cdk/lib/discount-expiry-notifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ export class DiscountExpiryNotifier extends GuStack {
app: appName,
functionName: `${appName}-get-subs-with-expiring-discounts-${this.stage}`,
runtime: nodeVersion,
environment: {
Stage: this.stage,
},
handler: 'getSubsWithExpiringDiscounts.handler',
fileName: `${appName}.zip`,
architecture: Architecture.ARM_64,
Expand All @@ -66,9 +63,6 @@ export class DiscountExpiryNotifier extends GuStack {
app: appName,
functionName: `${appName}-sub-is-active-${this.stage}`,
runtime: nodeVersion,
environment: {
Stage: this.stage,
},
handler: 'subIsActive.handler',
fileName: `${appName}.zip`,
architecture: Architecture.ARM_64,
Expand All @@ -90,9 +84,6 @@ export class DiscountExpiryNotifier extends GuStack {
app: appName,
functionName: `${appName}-build-email-payload-${this.stage}`,
runtime: nodeVersion,
environment: {
Stage: this.stage,
},
handler: 'buildEmailPayload.handler',
fileName: `${appName}.zip`,
architecture: Architecture.ARM_64,
Expand All @@ -106,9 +97,6 @@ export class DiscountExpiryNotifier extends GuStack {
app: appName,
functionName: `${appName}-initiate-email-send-${this.stage}`,
runtime: nodeVersion,
environment: {
Stage: this.stage,
},
handler: 'initiateEmailSend.handler',
fileName: `${appName}.zip`,
architecture: Architecture.ARM_64,
Expand All @@ -123,14 +111,14 @@ export class DiscountExpiryNotifier extends GuStack {
functionName: `${appName}-save-results-${this.stage}`,
runtime: nodeVersion,
environment: {
Stage: this.stage,
S3_BUCKET: bucket.bucketName,
},
handler: 'saveResults.handler',
fileName: `${appName}.zip`,
architecture: Architecture.ARM_64,
initialPolicy: [
new PolicyStatement({
actions: ['s3:PutObject'],
actions: ['s3:GetObject', 's3:PutObject'],
resources: [bucket.arnForObjects('*')],
}),
],
Expand Down Expand Up @@ -180,7 +168,7 @@ export class DiscountExpiryNotifier extends GuStack {

const emailSendsProcessingMap = new Map(this, 'Email sends processor map', {
maxConcurrency: 10,
itemsPath: JsonPath.stringAt('$.discountsToProcess'),
itemsPath: JsonPath.stringAt('$.expiringDiscountsToProcess'),
parameters: {
item: JsonPath.stringAt('$$.Map.Item.Value'),
},
Expand All @@ -193,7 +181,7 @@ export class DiscountExpiryNotifier extends GuStack {
subIsActiveLambdaTask.next(
isSubActiveChoice
.when(
Condition.booleanEquals('$.isActive', true),
Condition.stringEquals('$.status', 'Active'),
buildEmailPayloadLambdaTask.next(initiateEmailSendLambdaTask),
)
.otherwise(
Expand Down
1 change: 1 addition & 0 deletions handlers/discount-expiry-notifier/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"@types/aws-sdk": "^2.7.4"
},
"dependencies": {
"@aws-sdk/client-s3": "3.665.0",
"@google-cloud/bigquery": "^7.9.1",
"aws-sdk": "^2.1692.0",
"google-auth-library": "^9.15.0",
Expand Down
2 changes: 1 addition & 1 deletion handlers/discount-expiry-notifier/src/bigquery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ export const runQuery = async (
AND sub.is_latest_version = TRUE
AND sub.status = 'Active'
AND DATE_ADD(charge.effective_start_date, INTERVAL charge.up_to_periods MONTH) = DATE_ADD(CURRENT_DATE(), INTERVAL 32 DAY)
LIMIT 2
AND sub.name IN ('A-S02282302', 'A-S02282308')
)
SELECT
exp.sub_name as subName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,7 @@ export const handler = async () => {
const result = await runQuery(authClient);
console.log('result: ', result);

return result;
return {
expiringDiscountsToProcess: result,
};
};
48 changes: 47 additions & 1 deletion handlers/discount-expiry-notifier/src/handlers/saveResults.ts
Original file line number Diff line number Diff line change
@@ -1 +1,47 @@
export const handler = () => {};
import { getIfDefined } from '@modules/nullAndUndefined';
import { uploadFileToS3 } from '../s3';

type ExpiringDiscountToProcess = {
subName: string;
firstName: string;
paymentAmount: number;
paymentFrequency: string;
nextPaymentDate: string;
};

type ExpiringDiscountProcessingAttempt = {
status: string;
};

type LambdaInput = {
expiringDiscountsToProcess: ExpiringDiscountToProcess[];
expiringDiscountProcessingAttempts: ExpiringDiscountProcessingAttempt[];
};

export const handler = async (event: LambdaInput) => {
const bucketName = getIfDefined<string>(
process.env.S3_BUCKET,
'S3_BUCKET environment variable not set',
);

const getCurrentDateString = (): string => {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0'); // Months are 0-based
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
};

const filePath = getCurrentDateString();

//TODO add a precheck to find out if the file exists already and either append or create a separate file
await uploadFileToS3({
bucketName,
filePath,
content: JSON.stringify(event),
});

return {
filePath,
};
};
61 changes: 61 additions & 0 deletions handlers/discount-expiry-notifier/src/s3.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import type {
GetObjectCommandInput,
PutObjectCommandInput,
} from '@aws-sdk/client-s3';
import {
GetObjectCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';

const s3Client = new S3Client({ region: 'eu-west-1' });

export const uploadFileToS3 = async ({
bucketName,
filePath,
content,
}: {
bucketName: PutObjectCommandInput['Bucket'];
filePath: PutObjectCommandInput['Key'];
content: PutObjectCommandInput['Body'];
}) => {
try {
const command = new PutObjectCommand({
Bucket: bucketName,
Key: filePath,
Body: content,
});
const response = await s3Client.send(command);
return response;
} catch (error) {
console.error(error);
throw error;
}
};

export const getFileFromS3 = async ({
bucketName,
filePath,
}: {
bucketName: GetObjectCommandInput['Bucket'];
filePath: GetObjectCommandInput['Key'];
}) => {
try {
const command = new GetObjectCommand({
Bucket: bucketName,
Key: filePath,
});

const response = await s3Client.send(command);
const fileContent = response.Body?.transformToString();

if (!fileContent) {
throw new Error('File is empty');
}

return fileContent;
} catch (error) {
console.error(error);
throw error;
}
};
Loading

0 comments on commit 473c357

Please sign in to comment.