Pf/reuters poller

cdk/lib/__snapshots__/riffraff.test.ts.snap

@@ -45,6 +45,20 @@ deployments:
       fileName: poller-lambdas.zip
     actions:
       - uploadLambda
+  lambda-upload-eu-west-1-editorial-feeds-reuters_poller_lambda:
+    type: aws-lambda
+    stacks:
+      - editorial-feeds
+    regions:
+      - eu-west-1
+    app: reuters_poller_lambda
+    contentDirectory: poller-lambdas
+    parameters:
+      bucketSsmLookup: true
+      lookupByTags: true
+      fileName: poller-lambdas.zip
+    actions:
+      - uploadLambda
   asg-upload-eu-west-1-editorial-feeds-newswires:
     type: autoscaling
     actions:
@@ -79,6 +93,7 @@ deployments:
     dependencies:
       - lambda-upload-eu-west-1-editorial-feeds-ingestion-lambda
       - lambda-upload-eu-west-1-editorial-feeds-apPoller_poller_lambda
+      - lambda-upload-eu-west-1-editorial-feeds-reuters_poller_lambda
       - asg-upload-eu-west-1-editorial-feeds-newswires
       - cfn-eu-west-1-editorial-feeds-wires-feeds
   lambda-update-eu-west-1-editorial-feeds-ingestion-lambda:
@@ -113,6 +128,22 @@ deployments:
       - updateLambda
     dependencies:
       - cfn-eu-west-1-editorial-feeds-newswires
+  lambda-update-eu-west-1-editorial-feeds-reuters_poller_lambda:
+    type: aws-lambda
+    stacks:
+      - editorial-feeds
+    regions:
+      - eu-west-1
+    app: reuters_poller_lambda
+    contentDirectory: poller-lambdas
+    parameters:
+      bucketSsmLookup: true
+      lookupByTags: true
+      fileName: poller-lambdas.zip
+    actions:
+      - updateLambda
+    dependencies:
+      - cfn-eu-west-1-editorial-feeds-newswires
   asg-update-eu-west-1-editorial-feeds-newswires:
     type: autoscaling
     actions:

cdk/lib/constructs/pollerLambda.ts

@@ -103,6 +103,7 @@ export class PollerLambda {
 		});
 
 		secret.grantRead(lambda);
+		secret.grantWrite(lambda);
 
 		// wire up lambda to process its own queue
 		lambda.addEventSource(new SqsEventSource(lambdaQueue, { batchSize: 1 }));

package.json

@@ -1,55 +1,69 @@
 {
-	"name": "newswires",
-	"version": "1.0.0",
-	"description": "Newswires feed app",
-	"workspaces": [
-		"cdk",
-		"ingestion-lambda",
-		"poller-lambdas"
-	],
-	"scripts": {
-		"test": "npm run test --workspaces --if-present",
-		"synth": "npm run synth --workspace=cdk",
-		"typecheck": "npm run typecheck --workspaces",
-		"format": "prettier --write --ignore-path .gitignore \"./**/*{.js,.ts,.json,.md,.yml}\"",
-		"build": "npm run build --workspaces --if-present",
-		"lint": "npm run lint --workspaces",
-		"lint:ci": "npm run lint:ci --workspaces",
-		"dev": "npm run dev --workspace=ingestion-lambda"
-	},
-	"repository": {
-		"type": "git",
-		"url": "github.com/guardian/newswires"
-	},
-	"devDependencies": {
-		"@guardian/eslint-config-typescript": "8.0.0",
-		"@guardian/prettier": "5.0.0",
-		"@guardian/tsconfig": "1.0.0",
-		"@types/jest": "29.5.12",
-		"@types/node": "22.4.2",
-		"esbuild": "^0.24.0",
-		"eslint": "8.57.0",
-		"jest": "29.7.0",
-		"prettier": "3.3.3",
-		"ts-jest": "29.2.4",
-		"ts-node-dev": "^2.0.0",
-		"typescript": "5.1.6"
-	},
-	"prettier": "@guardian/prettier",
-	"eslintConfig": {
-		"extends": [
-			"@guardian/eslint-config-typescript"
-		],
-		"ignorePatterns": [
-			"**/*.js",
-			"node_modules",
-			"cdk.out",
-			".eslintrc.js",
-			"jest.config.js",
-			"**/generated/*"
-		]
-	},
-	"dependencies": {
-		"zod": "3.23.8"
-	}
+  "name": "newswires",
+  "version": "1.0.0",
+  "description": "Newswires feed app",
+  "workspaces": [
+    "cdk",
+    "ingestion-lambda",
+    "poller-lambdas"
+  ],
+  "scripts": {
+    "test": "npm run test --workspaces --if-present",
+    "synth": "npm run synth --workspace=cdk",
+    "typecheck": "npm run typecheck --workspaces",
+    "format": "prettier --write --ignore-path .gitignore \"./**/*{.js,.ts,.json,.md,.yml}\"",
+    "build": "npm run build --workspaces --if-present",
+    "lint": "npm run lint --workspaces",
+    "lint:ci": "npm run lint:ci --workspaces",
+    "dev": "npm run dev --workspace=ingestion-lambda"
+  },
+  "repository": {
+    "type": "git",
+    "url": "github.com/guardian/newswires"
+  },
+  "devDependencies": {
+    "@guardian/eslint-config-typescript": "8.0.0",
+    "@guardian/prettier": "5.0.0",
+    "@guardian/tsconfig": "1.0.0",
+    "@types/jest": "29.5.12",
+    "@types/node": "22.4.2",
+    "esbuild": "^0.24.0",
+    "eslint": "8.57.0",
+    "jest": "29.7.0",
+    "prettier": "3.3.3",
+    "ts-jest": "29.2.4",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "5.1.6"
+  },
+  "prettier": "@guardian/prettier",
+  "eslintConfig": {
+    "extends": [
+      "@guardian/eslint-config-typescript"
+    ],
+    "rules": {
+      "@typescript-eslint/no-unused-vars": [
+        "error",
+        {
+          "args": "all",
+          "argsIgnorePattern": "^_",
+          "caughtErrors": "all",
+          "caughtErrorsIgnorePattern": "^_",
+          "destructuredArrayIgnorePattern": "^_",
+          "varsIgnorePattern": "^_",
+          "ignoreRestSiblings": true
+        }
+      ]
+    },
+    "ignorePatterns": [
+      "**/*.js",
+      "node_modules",
+      "cdk.out",
+      ".eslintrc.js",
+      "jest.config.js",
+      "**/generated/*"
+    ]
+  },
+  "dependencies": {
+    "zod": "3.23.8"
+  }
 }

poller-lambdas/src/index.ts

@@ -1,24 +1,29 @@
-import { GetSecretValueCommand } from '@aws-sdk/client-secrets-manager';
+import {
+	GetSecretValueCommand,
+	PutSecretValueCommand,
+} from '@aws-sdk/client-secrets-manager';
 import type { SendMessageCommandInput } from '@aws-sdk/client-sqs';
 import { SendMessageCommand } from '@aws-sdk/client-sqs';
 import type { PollerId } from '../../shared/pollers';
 import { POLLER_LAMBDA_ENV_VAR_KEYS } from '../../shared/pollers';
 import { queueNextInvocation, secretsManager, sqs } from './aws';
 import { getEnvironmentVariableOrCrash } from './config';
 import { apPoller } from './pollers/AP/apPoller';
+import { reutersPoller } from './pollers/reuters/reutersPoller';
 import type { HandlerInputSqsPayload, PollFunction } from './types';
 import { isFixedFrequencyPollOutput } from './types';
 
 const pollerWrapper =
 	(pollerFunction: PollFunction) =>
 	async ({ Records }: HandlerInputSqsPayload) => {
 		const startTimeEpochMillis = Date.now();
+		const secretName = getEnvironmentVariableOrCrash(
+			POLLER_LAMBDA_ENV_VAR_KEYS.SECRET_NAME,
+		);
 		const secret = await secretsManager
 			.send(
 				new GetSecretValueCommand({
-					SecretId: getEnvironmentVariableOrCrash(
-						POLLER_LAMBDA_ENV_VAR_KEYS.SECRET_NAME,
-					),
+					SecretId: secretName,
 				}),
 			)
 			.then((_) => _.SecretString);
@@ -89,6 +94,17 @@ const pollerWrapper =
 							MessageBody: output.valueForNextPoll,
 						});
 					}
+
+					if (output.newSecretValue) {
+						// set new value in secrets manager
+						console.log(`Updating secret value for ${secretName}`);
+						await secretsManager.send(
+							new PutSecretValueCommand({
+								SecretId: secretName,
+								SecretString: output.newSecretValue,
+							}),
+						);
+					}
 				})
 				.catch((error) => {
 					console.error('FAILED', error);
@@ -102,6 +118,7 @@ export const handlers = {
 	// EXAMPLE_long_polling: pollerWrapper(EXAMPLE_long_polling),
 	// EXAMPLE_fixed_frequency: pollerWrapper(EXAMPLE_fixed_frequency),
 	apPoller: pollerWrapper(apPoller),
+	reuters: pollerWrapper(reutersPoller),
 } satisfies Record<
 	PollerId,
 	(sqsEvent: HandlerInputSqsPayload) => Promise<void>

poller-lambdas/src/pollers/reuters/auth.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod';
+
+const AuthSchema = z.object({
+	access_token: z.string(),
+	expires_in: z.number(),
+	token_type: z.string(),
+});
+
+const scopes =
+	'https://api.thomsonreuters.com/auth/reutersconnect.contentapi.read https://api.thomsonreuters.com/auth/reutersconnect.contentapi.write';
+const authUrl = 'https://auth.thomsonreuters.com/oauth/token';
+const grantType = 'client_credentials';
+const audience = '7a14b6a2-73b8-4ab2-a610-80fb9f40f769';
+
+export async function auth(
+	clientId: string,
+	clientSecret: string,
+): Promise<string> {
+	const req = new Request(authUrl, {
+		method: 'POST',
+		headers: {
+			'content-type': 'application/x-www-form-urlencoded',
+		},
+		body: `grant_type=${grantType}&client_id=${clientId}&client_secret=${clientSecret}&audience=${audience}&scope=${encodeURIComponent(scopes)}`,
+	});
+	try {
+		console.log('Requesting new auth token from Reuters');
+		const response = await fetch(req);
+		const data = (await response.json()) as unknown;
+		const { access_token, expires_in } = AuthSchema.parse(data);
+		console.log(
+			`Received new auth token from Reuters, expires in ${expires_in} seconds`,
+		);
+		return access_token;
+	} catch (error) {
+		console.error(error);
+		throw new Error('Failed to get auth token from Reuters');
+	}
+}