feature(general): import and process payments files from postfinance …

…ftp server (#764)

.github/workflows/deployment.yml

@@ -82,6 +82,10 @@ jobs:
         run: |
           echo POSTFINANCE_EMAIL_PASSWORD=${{ secrets.POSTFINANCE_EMAIL_PASSWORD }} > .env
           echo POSTFINANCE_PAYMENTS_FILES_BUCKET=${{ (inputs.project == 'social-income-prod' && vars.POSTFINANCE_PAYMENTS_FILES_BUCKET) || vars.POSTFINANCE_PAYMENTS_FILES_BUCKET_STAGING }} >> .env
+          echo POSTFINANCE_FTP_HOST=${{ vars.POSTFINANCE_FTP_HOST }} >> .env
+          echo POSTFINANCE_FTP_PORT=${{ vars.POSTFINANCE_FTP_PORT }} >> .env
+          echo POSTFINANCE_FTP_USER=${{ vars.POSTFINANCE_FTP_USER }} >> .env
+          echo POSTFINANCE_FTP_RSA_PRIVATE_KEY=${{ secrets.POSTFINANCE_FTP_RSA_PRIVATE_KEY }} >> .env
           echo STRIPE_API_READ_KEY=${{ (inputs.project == 'social-income-prod' && secrets.STRIPE_API_READ_KEY) || secrets.STRIPE_API_READ_KEY_STAGING }} >> .env
           echo STRIPE_WEBHOOK_SECRET=${{ (inputs.project == 'social-income-prod' && secrets.STRIPE_WEBHOOK_SECRET) || secrets.STRIPE_WEBHOOK_SECRET_STAGING }} >> .env
           echo NOTIFICATION_EMAIL_USER=${{ secrets.NOTIFICATION_EMAIL_USER }} >> .env

functions/.env.sample

@@ -1,6 +1,16 @@
 POSTFINANCE_EMAIL_USER=[email protected]
 POSTFINANCE_EMAIL_PASSWORD=password
 POSTFINANCE_PAYMENTS_FILES_BUCKET=postfinance-payments-files
+POSTFINANCE_FTP_HOST=ftp.postfinance.example
+POSTFINANCE_FTP_PORT=21
+POSTFINANCE_FTP_USER=example
+POSTFINANCE_FTP_RSA_PRIVATE_KEY="
+-----BEGIN RSA PRIVATE KEY-----
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+...
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+-----END RSA PRIVATE KEY-----
+"
 
 # To work with the Twilio API locally, configure the test credentials from Twilio in your .env file
 TWILIO_SID=ACXXXXXXXXXXXXXXXXXXXX

functions/package.json

@@ -31,6 +31,7 @@
 		"typescript": "^5.3.2"
 	},
 	"dependencies": {
+		"@types/ssh2-sftp-client": "^9.0.3",
 		"@xmldom/xmldom": "^0.8.10",
 		"axios": "^1.6.2",
 		"dotenv": "^16.3.1",
@@ -46,6 +47,7 @@
 		"mjml": "^4.14.1",
 		"nodemailer": "^6.9.9",
 		"pdfkit": "^0.14.0",
+		"ssh2-sftp-client": "^10.0.3",
 		"stripe": "^14.5.0",
 		"tmp-promise": "^3.0.3",
 		"twilio": "^4.19.0",

functions/src/config.ts

@@ -12,6 +12,10 @@ export const POSTFINANCE_EMAIL_USER = process.env.POSTFINANCE_EMAIL_USER!;
 export const POSTFINANCE_EMAIL_PASSWORD = process.env.POSTFINANCE_EMAIL_PASSWORD!;
 
 export const POSTFINANCE_PAYMENTS_FILES_BUCKET = process.env.POSTFINANCE_PAYMENTS_FILES_BUCKET!;
+export const POSTFINANCE_FTP_RSA_PRIVATE_KEY = process.env.POSTFINANCE_FTP_RSA_PRIVATE_KEY!;
+export const POSTFINANCE_FTP_HOST = process.env.POSTFINANCE_FTP_HOST!;
+export const POSTFINANCE_FTP_PORT = process.env.POSTFINANCE_FTP_PORT!;
+export const POSTFINANCE_FTP_USER = process.env.POSTFINANCE_FTP_USER!;
 
 export const NOTIFICATION_EMAIL_USER = process.env.NOTIFICATION_EMAIL_USER!;
 export const NOTIFICATION_EMAIL_PASSWORD = process.env.NOTIFICATION_EMAIL_PASSWORD!;

functions/src/cron/index.ts

@@ -1,5 +1,7 @@
 import importExchangeRatesFunction from './exchange-rate-import';
 import importBalanceMailFunction from './postfinance-balance-import';
+import importPostfinancePaymentsFilesFunction from './postfinance-payments-files-import';
 
 export const importBalanceMail = importBalanceMailFunction;
+export const importPostfinancePaymentsFiles = importPostfinancePaymentsFilesFunction;
 export const importExchangeRates = importExchangeRatesFunction;

functions/src/cron/postfinance-payments-files-import/index.ts

@@ -0,0 +1,8 @@
+import { onSchedule } from 'firebase-functions/v2/scheduler';
+import { POSTFINANCE_PAYMENTS_FILES_BUCKET } from '../../config';
+import { PostfinancePaymentsFileHandler } from '../../utils/PostfinancePaymentsFileHandler';
+
+export default onSchedule('0 * * * *', async () => {
+	const paymentsFileHandler = new PostfinancePaymentsFileHandler(POSTFINANCE_PAYMENTS_FILES_BUCKET);
+	await paymentsFileHandler.importPaymentFiles();
+});

functions/src/storage/index.ts

@@ -1,3 +1,3 @@
 import onPostfinancePaymentsFileCreatedFunction from './postfinance-payments-files';
 
-export const onPostfinancePaymentsFileCreated = onPostfinancePaymentsFileCreatedFunction;
+export const onPostfinancePaymentsFileUploaded = onPostfinancePaymentsFileCreatedFunction;

functions/src/storage/postfinance-payments-files/index.ts

@@ -1,8 +1,8 @@
 import { onObjectFinalized } from 'firebase-functions/v2/storage';
 import { POSTFINANCE_PAYMENTS_FILES_BUCKET } from '../../config';
-import { PostfinancePaymentsFileImporter } from './PostfinancePaymentsFileImporter';
+import { PostfinancePaymentsFileHandler } from '../../utils/PostfinancePaymentsFileHandler';
 
 export default onObjectFinalized({ bucket: POSTFINANCE_PAYMENTS_FILES_BUCKET }, async (event) => {
-	const paymentsFileImporter = new PostfinancePaymentsFileImporter(POSTFINANCE_PAYMENTS_FILES_BUCKET);
-	paymentsFileImporter.processPaymentFile(event.data.name);
+	const paymentsFileHandler = new PostfinancePaymentsFileHandler(POSTFINANCE_PAYMENTS_FILES_BUCKET);
+	await paymentsFileHandler.processPaymentFile(event.data.name);
 });

functions/src/utils/PostfinancePaymentsFileHandler.ts

@@ -0,0 +1,123 @@
+import xmldom from '@xmldom/xmldom';
+import { DateTime } from 'luxon';
+import SFTPClient from 'ssh2-sftp-client';
+import { withFile } from 'tmp-promise';
+import xpath from 'xpath';
+import { FirestoreAdmin } from '../../../shared/src/firebase/admin/FirestoreAdmin';
+import { StorageAdmin } from '../../../shared/src/firebase/admin/StorageAdmin';
+import { toFirebaseAdminTimestamp } from '../../../shared/src/firebase/admin/utils';
+import {
+	BankWireContribution,
+	CONTRIBUTION_FIRESTORE_PATH,
+	ContributionSourceKey,
+	StatusKey,
+} from '../../../shared/src/types/contribution';
+import { Currency } from '../../../shared/src/types/currency';
+import { USER_FIRESTORE_PATH, User } from '../../../shared/src/types/user';
+import {
+	POSTFINANCE_FTP_HOST,
+	POSTFINANCE_FTP_PORT,
+	POSTFINANCE_FTP_RSA_PRIVATE_KEY,
+	POSTFINANCE_FTP_USER,
+} from '../config';
+
+export class PostfinancePaymentsFileHandler {
+	private storageAdmin: StorageAdmin;
+	private firestoreAdmin: FirestoreAdmin;
+	private readonly bucketName;
+	private readonly xmlSelectExpression = '//ns:BkToCstmrDbtCdtNtfctn/ns:Ntfctn/ns:Ntry/ns:NtryDtls/ns:TxDtls';
+
+	constructor(bucketName: string) {
+		this.storageAdmin = new StorageAdmin();
+		this.firestoreAdmin = new FirestoreAdmin();
+		this.bucketName = bucketName;
+	}
+
+	/**
+	 * Imports payment files from the Postfinance SFTP server to the payments files storage bucket
+	 */
+	async importPaymentFiles() {
+		const sftp = new SFTPClient();
+		const bucket = this.storageAdmin.storage.bucket(this.bucketName);
+		const files = (await bucket.getFiles())[0].map((file) => file.name);
+
+		await sftp.connect({
+			host: POSTFINANCE_FTP_HOST,
+			port: Number(POSTFINANCE_FTP_PORT),
+			username: POSTFINANCE_FTP_USER,
+			privateKey: POSTFINANCE_FTP_RSA_PRIVATE_KEY,
+		});
+		const sftpFiles = await sftp.list('/yellow-net-reports');
+		for (let file of sftpFiles) {
+			if (files.includes(file.name)) {
+				console.info(`Skipped copying file ${file.name} because it already exists in ${this.bucketName} bucket`);
+				continue;
+			}
+
+			await withFile(async ({ path }) => {
+				await sftp.get(`/yellow-net-reports/${file.name}`, path);
+				await this.storageAdmin.uploadFile({
+					bucket: bucket,
+					sourceFilePath: path,
+					destinationFilePath: file.name,
+				});
+				console.info(`Successfully copied ${file.name} to ${this.bucketName} bucket`);
+			});
+		}
+	}
+
+	/**
+	 * Processes a payment file from the payments files storage bucket
+	 * @param fileName The name of the file to process
+	 */
+	processPaymentFile(fileName: string) {
+		if (!fileName.startsWith('camt.054_P_')) {
+			console.info(`Skipped processing ${fileName} because it does not contain relevant payment data`);
+			return;
+		}
+
+		this.storageAdmin.storage
+			.bucket(this.bucketName)
+			.file(fileName)
+			.download()
+			.then(async (data) => {
+				const xmlString = data[0].toString('utf8');
+				const xmlDoc = new xmldom.DOMParser().parseFromString(xmlString, 'text/xml');
+				const select = xpath.useNamespaces({ ns: 'urn:iso:std:iso:20022:tech:xsd:camt.054.001.08' });
+				const nodes = select(this.xmlSelectExpression, xmlDoc) as Node[];
+
+				for (let node of nodes) {
+					const transactionId = select('string(//ns:Refs/ns:EndToEndId)', node) as string;
+					const referenceId = select('string(//ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref)', node) as string;
+					const userReferenceId = parseInt(referenceId.slice(7, 20));
+					console.info(
+						`Processing transaction ${transactionId} with reference ${referenceId} for user ${userReferenceId}`,
+					);
+
+					const contribution: BankWireContribution = {
+						reference_id: referenceId,
+						transaction_id: transactionId,
+						monthly_interval: parseInt(referenceId.slice(20, 22)),
+						currency: (select('string(//ns:Amt/@Ccy)', node) as string).toUpperCase() as Currency,
+						amount: parseFloat(select('string(//ns:Amt)', node) as string),
+						amount_chf: parseFloat(select('string(//ns:Amt)', node) as string),
+						fees_chf: 0,
+						status: StatusKey.SUCCEEDED,
+						created: toFirebaseAdminTimestamp(DateTime.now()),
+						source: ContributionSourceKey.WIRE_TRANSFER,
+						raw_content: node.toString(),
+					};
+
+					const user = await this.firestoreAdmin.findFirst<User>(USER_FIRESTORE_PATH, (q) =>
+						q.where('payment_reference_id', '==', userReferenceId),
+					);
+
+					if (user) {
+						await user?.ref.collection(CONTRIBUTION_FIRESTORE_PATH).doc(transactionId).set(contribution);
+					} else {
+						await this.firestoreAdmin.collection(CONTRIBUTION_FIRESTORE_PATH).doc(transactionId).set(contribution);
+					}
+				}
+			});
+	}
+}