From 48f860621b29a28e0771f2787e9c390523cbd475 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sat, 16 Nov 2024 21:52:58 +0200 Subject: [PATCH 01/18] feat: tm monitoring changes for better investigation --- pnpm-lock.yaml | 7 +- services/workflows-service/jest.config.cjs | 1 + services/workflows-service/package.json | 1 + .../workflows-service/prisma/data-migrations | 2 +- .../migration.sql | 15 ++ .../workflows-service/prisma/schema.prisma | 12 ++ .../scripts/alerts/generate-alerts.ts | 81 ++++++--- .../src/alert/alert.controller.external.ts | 41 +++++ .../src/alert/alert.service.ts | 21 ++- services/workflows-service/src/alert/types.ts | 12 +- .../controllers/case-management.controller.ts | 2 +- .../data-analytics/data-analytics.service.ts | 159 +++++++++++++++++- .../src/data-analytics/types.ts | 19 ++- services/workflows-service/src/global.d.ts | 3 +- .../src/project/project-scope.service.ts | 29 +++- .../transaction.controller.external.ts | 132 +++++++++++---- .../src/transaction/transaction.repository.ts | 76 +++++++-- .../src/transaction/transaction.service.ts | 10 +- 18 files changed, 524 insertions(+), 99 deletions(-) create mode 100644 services/workflows-service/prisma/migrations/20241021185057_add_alerts_counterparty_relation_for_advanced_filtering/migration.sql diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37c3b20601..edab749ea1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2744,6 +2744,9 @@ importers: deep-diff: specifier: ^1.0.2 version: 1.0.2 + deepmerge: + specifier: ^4.3.0 + version: 4.3.1 file-type: specifier: ^16.5.4 version: 16.5.4 @@ -25309,7 +25312,7 @@ packages: eslint-import-resolver-webpack: optional: true dependencies: - '@typescript-eslint/parser': 5.62.0(eslint@8.54.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.62.0(eslint@8.54.0)(typescript@4.9.3) debug: 3.2.7 eslint: 8.54.0 eslint-import-resolver-node: 0.3.9 @@ -25474,7 +25477,7 @@ packages: '@typescript-eslint/parser': optional: true dependencies: - '@typescript-eslint/parser': 5.62.0(eslint@8.54.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.62.0(eslint@8.54.0)(typescript@4.9.3) array-includes: 3.1.7 array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 diff --git a/services/workflows-service/jest.config.cjs b/services/workflows-service/jest.config.cjs index a0ff659621..745c2bf710 100644 --- a/services/workflows-service/jest.config.cjs +++ b/services/workflows-service/jest.config.cjs @@ -1,6 +1,7 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', + testTimeout: 30000, modulePathIgnorePatterns: ['/dist/'], testRegex: '(/__tests__/.*|(\\.|/)(unit|e2e|intg)\\.test)\\.ts$', moduleNameMapper: { diff --git a/services/workflows-service/package.json b/services/workflows-service/package.json index a063145c96..28e887890c 100644 --- a/services/workflows-service/package.json +++ b/services/workflows-service/package.json @@ -86,6 +86,7 @@ "csv-parse": "^5.5.6", "dayjs": "^1.11.6", "deep-diff": "^1.0.2", + "deepmerge": "^4.3.0", "file-type": "^16.5.4", "helmet": "^6.0.1", "i18n-iso-countries": "^7.6.0", diff --git a/services/workflows-service/prisma/data-migrations b/services/workflows-service/prisma/data-migrations index a0c3055b4d..f9008a19a5 160000 --- a/services/workflows-service/prisma/data-migrations +++ b/services/workflows-service/prisma/data-migrations @@ -1 +1 @@ -Subproject commit a0c3055b4df212f7d4cf13a200593fe4de2ae909 +Subproject commit f9008a19a5430cead09300ac3e519d7189e8576c diff --git a/services/workflows-service/prisma/migrations/20241021185057_add_alerts_counterparty_relation_for_advanced_filtering/migration.sql b/services/workflows-service/prisma/migrations/20241021185057_add_alerts_counterparty_relation_for_advanced_filtering/migration.sql new file mode 100644 index 0000000000..f99421860a --- /dev/null +++ b/services/workflows-service/prisma/migrations/20241021185057_add_alerts_counterparty_relation_for_advanced_filtering/migration.sql @@ -0,0 +1,15 @@ +-- AlterTable +ALTER TABLE "Alert" ADD COLUMN "counterpartyBeneficiaryId" TEXT, +ADD COLUMN "counterpartyOriginatorId" TEXT; + +-- CreateIndex +CREATE INDEX "Alert_counterpartyOriginatorId_idx" ON "Alert"("counterpartyOriginatorId"); + +-- CreateIndex +CREATE INDEX "Alert_counterpartyBeneficiaryId_idx" ON "Alert"("counterpartyBeneficiaryId"); + +-- AddForeignKey +ALTER TABLE "Alert" ADD CONSTRAINT "Alert_counterpartyOriginatorId_fkey" FOREIGN KEY ("counterpartyOriginatorId") REFERENCES "Counterparty"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Alert" ADD CONSTRAINT "Alert_counterpartyBeneficiaryId_fkey" FOREIGN KEY ("counterpartyBeneficiaryId") REFERENCES "Counterparty"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/services/workflows-service/prisma/schema.prisma b/services/workflows-service/prisma/schema.prisma index 66bfad1b06..1bf8b994de 100644 --- a/services/workflows-service/prisma/schema.prisma +++ b/services/workflows-service/prisma/schema.prisma @@ -834,9 +834,15 @@ model Alert { workflowRuntimeDataId String? workflowRuntimeData WorkflowRuntimeData? @relation(fields: [workflowRuntimeDataId], references: [id], onUpdate: Cascade, onDelete: NoAction) + // TODO: Remove this field after data migration counterpartyId String? counterparty Counterparty? @relation(fields: [counterpartyId], references: [id]) + counterpartyOriginatorId String? + counterpartyBeneficiaryId String? + counterpartyOriginator Counterparty? @relation(name: "counterpartyAlertOriginator", fields: [counterpartyOriginatorId], references: [id]) + counterpartyBeneficiary Counterparty? @relation(name: "counterpartyAlertBeneficiary", fields: [counterpartyBeneficiaryId], references: [id]) + businessId String? business Business? @relation(fields: [businessId], references: [id]) @@ -845,6 +851,9 @@ model Alert { @@index([alertDefinitionId]) @@index([counterpartyId]) @@index([createdAt(sort: Desc)]) + + @@index([counterpartyOriginatorId]) + @@index([counterpartyBeneficiaryId]) } enum RiskCategory { @@ -888,6 +897,9 @@ model Counterparty { benefitingTransactions TransactionRecord[] @relation("BenefitingCounterparty") alerts Alert[] + alertsBenefiting Alert[] @relation("counterpartyAlertBeneficiary") + alertsOriginating Alert[] @relation("counterpartyAlertOriginator") + projectId String project Project @relation(fields: [projectId], references: [id]) diff --git a/services/workflows-service/scripts/alerts/generate-alerts.ts b/services/workflows-service/scripts/alerts/generate-alerts.ts index d1aa16ec3e..ecfd4bdfaa 100644 --- a/services/workflows-service/scripts/alerts/generate-alerts.ts +++ b/services/workflows-service/scripts/alerts/generate-alerts.ts @@ -44,21 +44,25 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PAY_HCA_CC', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId'], options: { havingAggregate: AggregateType.SUM, + groupBy: ['counterpartyBeneficiaryId'], + direction: TransactionDirection.inbound, + excludedCounterparty: { counterpartyBeneficiaryIds: ['9999999999999999', '999999______9999'], counterpartyOriginatorIds: [], }, + paymentMethods: [PaymentMethod.credit_card], excludePaymentMethods: false, timeAmount: SEVEN_DAYS, timeUnit: TIME_UNITS.days, amountThreshold: 1000, - groupBy: ['counterpartyBeneficiaryId'], }, }, }, @@ -70,9 +74,11 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PAY_HCA_APM', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId'], options: { havingAggregate: AggregateType.SUM, + groupBy: ['counterpartyBeneficiaryId'], direction: TransactionDirection.inbound, @@ -88,8 +94,6 @@ export const ALERT_DEFINITIONS = { timeUnit: TIME_UNITS.days, amountThreshold: 1000, - - groupBy: ['counterpartyBeneficiaryId'], }, }, }, @@ -101,12 +105,14 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'STRUC_CC', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId'], options: { havingAggregate: AggregateType.COUNT, groupBy: ['counterpartyBeneficiaryId'], direction: TransactionDirection.inbound, + excludedCounterparty: { counterpartyBeneficiaryIds: ['9999999999999999', '999999______9999'], counterpartyOriginatorIds: [], @@ -131,7 +137,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'STRUC_APM', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId'], options: { havingAggregate: AggregateType.COUNT, groupBy: ['counterpartyBeneficiaryId'], @@ -159,12 +166,14 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HCAI_CC', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId', 'counterpartyOriginatorId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], options: { havingAggregate: AggregateType.SUM, groupBy: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], direction: TransactionDirection.inbound, + excludedCounterparty: { counterpartyBeneficiaryIds: ['9999999999999999', '999999______9999'], counterpartyOriginatorIds: [], @@ -188,7 +197,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HACI_APM', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId', 'counterpartyOriginatorId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], options: { havingAggregate: AggregateType.SUM, groupBy: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], @@ -217,7 +227,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HVIC_CC', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId', 'counterpartyOriginatorId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], options: { havingAggregate: AggregateType.COUNT, groupBy: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], @@ -246,7 +257,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HVIC_CC', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId', 'counterpartyOriginatorId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], options: { havingAggregate: AggregateType.COUNT, groupBy: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], @@ -275,7 +287,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'CHVC_C', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyOriginatorId'], options: { transactionType: [TransactionRecordType.chargeback], paymentMethods: [PaymentMethod.credit_card], @@ -297,7 +310,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'SHCAC_C', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyOriginatorId'], options: { transactionType: [TransactionRecordType.chargeback], paymentMethods: [PaymentMethod.credit_card], @@ -319,7 +333,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'CHCR_C', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyOriginatorId'], options: { transactionType: [TransactionRecordType.refund], paymentMethods: [PaymentMethod.credit_card], @@ -341,7 +356,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'SHCAR_C', fnName: 'evaluateTransactionsAgainstDynamicRules', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', + subjects: ['counterpartyOriginatorId'], options: { transactionType: [TransactionRecordType.refund], paymentMethods: [PaymentMethod.credit_card], @@ -366,7 +382,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HPC', fnName: 'evaluateHighTransactionTypePercentage', - subjects: ['counterpartyId'], + fnInvestigationName: undefined, + subjects: ['counterpartyOriginatorId'], options: { transactionType: TransactionRecordType.chargeback, subjectColumn: 'counterpartyOriginatorId', @@ -384,7 +401,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'TLHAICC', fnName: 'evaluateTransactionAvg', - subjects: ['counterpartyId'], + subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, minimumCount: 2, @@ -407,7 +424,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'TLHAIAPM', fnName: 'evaluateTransactionAvg', - subjects: ['counterpartyId'], + subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, minimumCount: 2, @@ -430,7 +447,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PGAICT', fnName: 'evaluateTransactionAvg', - subjects: ['counterpartyId'], + subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, minimumCount: 2, @@ -454,7 +471,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PGAIAPM', fnName: 'evaluateTransactionAvg', - subjects: ['counterpartyId'], + subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, minimumCount: 2, @@ -518,11 +535,11 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HVHAI_APM', fnName: 'evaluateHighVelocityHistoricAverage', - subjects: ['counterpartyId'], + subjects: ['counterpartyBeneficiaryId'], options: { - transactionDirection: TransactionDirection.inbound, minimumCount: 3, transactionFactor: 2, + transactionDirection: TransactionDirection.inbound, paymentMethod: { value: PaymentMethod.credit_card, operator: '!=', @@ -622,7 +639,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'DSTA_CC', fnName: 'evaluateDailySingleTransactionAmount', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateDailySingleTransactionAmount', + subjects: ['counterpartyBeneficiaryId'], options: { ruleType: 'amount', @@ -635,6 +653,8 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, + + subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -645,7 +665,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'DSTA_APM', fnName: 'evaluateDailySingleTransactionAmount', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateDailySingleTransactionAmount', + subjects: ['counterpartyBeneficiaryId'], options: { ruleType: 'amount', @@ -658,6 +679,8 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, + + subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -668,7 +691,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'DMT_CC', fnName: 'evaluateDailySingleTransactionAmount', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateDailySingleTransactionAmount', + subjects: ['counterpartyBeneficiaryId'], options: { ruleType: 'count', @@ -681,6 +705,8 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, + + subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -691,7 +717,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'DMT_APM', fnName: 'evaluateDailySingleTransactionAmount', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateDailySingleTransactionAmount', + subjects: ['counterpartyBeneficiaryId'], options: { ruleType: 'count', @@ -704,6 +731,8 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, + + subjectColumn: 'counterpartyBeneficiaryId', }, }, }, diff --git a/services/workflows-service/src/alert/alert.controller.external.ts b/services/workflows-service/src/alert/alert.controller.external.ts index 30d880edb1..6ba09567a9 100644 --- a/services/workflows-service/src/alert/alert.controller.external.ts +++ b/services/workflows-service/src/alert/alert.controller.external.ts @@ -76,6 +76,47 @@ export class AlertControllerExternal { avatarUrl: true, }, }, + counterpartyOriginator: { + select: { + id: true, + business: { + select: { + id: true, + correlationId: true, + companyName: true, + }, + }, + endUser: { + select: { + id: true, + correlationId: true, + firstName: true, + lastName: true, + }, + }, + }, + }, + counterpartyBeneficiary: { + select: { + id: true, + business: { + select: { + id: true, + correlationId: true, + companyName: true, + }, + }, + endUser: { + select: { + id: true, + correlationId: true, + firstName: true, + lastName: true, + }, + }, + }, + }, + // TODO: remove this after migration counterparty: { select: { id: true, diff --git a/services/workflows-service/src/alert/alert.service.ts b/services/workflows-service/src/alert/alert.service.ts index 31a3d882ef..c255798928 100644 --- a/services/workflows-service/src/alert/alert.service.ts +++ b/services/workflows-service/src/alert/alert.service.ts @@ -16,6 +16,7 @@ import { AlertState, AlertStatus, MonitoringType, + Prisma, } from '@prisma/client'; import _ from 'lodash'; import { AlertExecutionStatus } from './consts'; @@ -329,17 +330,21 @@ export class AlertService { } createAlert( - alertDef: Partial, + alertDef: Partial & Required<{ projectId: AlertDefinition['projectId'] }>, subject: Array<{ [key: string]: unknown }>, executionRow: Record, additionalInfo?: Record, ) { + const mergedSubject = Object.assign({}, ...(subject || [])); + + const projectId = alertDef.projectId; + const now = new Date(); + return this.alertRepository.create({ data: { + projectId, alertDefinitionId: alertDef.id, - projectId: alertDef.projectId, severity: alertDef.defaultSeverity, - dataTimestamp: new Date(), state: AlertState.triggered, status: AlertStatus.new, additionalInfo: additionalInfo, @@ -347,10 +352,18 @@ export class AlertService { checkpoint: { hash: computeHash(executionRow), }, - subject: Object.assign({}, ...(subject || [])), + subject: mergedSubject, executionRow, + filters: this.dataAnalyticsService.getInvestigationFilter( + projectId, + alertDef.inlineRule as InlineRule, + mergedSubject, + ), } satisfies TExecutionDetails as InputJsonValue, ...Object.assign({}, ...(subject || [])), + updatedAt: now, + createdAt: now, + dataTimestamp: now, }, }); } diff --git a/services/workflows-service/src/alert/types.ts b/services/workflows-service/src/alert/types.ts index dbbeca2adc..436130208f 100644 --- a/services/workflows-service/src/alert/types.ts +++ b/services/workflows-service/src/alert/types.ts @@ -1,10 +1,18 @@ -import { Alert, AlertDefinition, Business, EndUser, User } from '@prisma/client'; +import { Alert, AlertDefinition, Business, EndUser, Prisma, User } from '@prisma/client'; + +// TODO: Remove counterpartyId from SubjectRecord +export type Subject = 'counterpartyOriginatorId' | 'counterpartyBeneficiaryId' | 'counterpartyId'; + +export type SubjectRecord = { + [key in Subject]?: string; +} & ({ counterpartyOriginatorId: string } | { counterpartyBeneficiaryId: string }); export type TExecutionDetails = { checkpoint: { hash: string; }; - subject: Array>; + subject: SubjectRecord; + filters: Prisma.TransactionRecordWhereInput; executionRow: unknown; }; diff --git a/services/workflows-service/src/case-management/controllers/case-management.controller.ts b/services/workflows-service/src/case-management/controllers/case-management.controller.ts index 6504936a34..4aadb3c0a0 100644 --- a/services/workflows-service/src/case-management/controllers/case-management.controller.ts +++ b/services/workflows-service/src/case-management/controllers/case-management.controller.ts @@ -71,7 +71,7 @@ export class CaseManagementController { @Get('transactions') async getTransactions(@CurrentProject() projectId: TProjectId) { - return this.transactionService.getAll({}, projectId); + return this.transactionService.getTransactions(projectId); } @Get('profiles/individuals') diff --git a/services/workflows-service/src/data-analytics/data-analytics.service.ts b/services/workflows-service/src/data-analytics/data-analytics.service.ts index b3c91e3077..8d0adaae6e 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.service.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.service.ts @@ -14,12 +14,13 @@ import { TMerchantGroupAverage, DailySingleTransactionAmountType, } from './types'; -import { AggregateType } from './consts'; +import { AggregateType, TIME_UNITS } from './consts'; import { calculateStartDate } from './utils'; -import { AlertSeverity, Prisma } from '@prisma/client'; +import { Alert, AlertSeverity, PaymentMethod, Prisma, TransactionRecordType } from '@prisma/client'; import { AppLoggerService } from '@/common/app-logger/app-logger.service'; import { isEmpty } from 'lodash'; import { MERCHANT_REPORT_TYPES_MAP, MerchantReportType } from '@/business-report/constants'; +import { SubjectRecord } from '@/alert/types'; const COUNTERPARTY_ORIGINATOR_JOIN_CLAUSE = Prisma.sql`JOIN "Counterparty" AS "cpOriginator" ON "tr"."counterpartyOriginatorId" = "cpOriginator"."id"`; const COUNTERPARTY_BENEFICIARY_JOIN_CLAUSE = Prisma.sql`JOIN "Counterparty" AS "cpBeneficiary" ON "tr"."counterpartyBeneficiaryId" = "cpBeneficiary"."id"`; @@ -429,7 +430,7 @@ export class DataAnalyticsService { const query: Prisma.Sql = Prisma.sql` WITH transactions AS ( SELECT - "tr"."counterpartyBeneficiaryId" AS "counterpartyId", + "tr"."counterpartyBeneficiaryId", count( CASE WHEN "tr"."transactionDate" >= CURRENT_DATE - INTERVAL '${Prisma.raw( `${timeAmount} ${timeUnit}`, @@ -575,7 +576,7 @@ export class DataAnalyticsService { HAVING COUNT(*) > ${minimumCount} ) SELECT - "tr"."counterpartyBeneficiaryId" AS "counterpartyId" + "tr"."counterpartyBeneficiaryId" as "counterpartyBeneficiaryId" FROM "TransactionRecord" tr JOIN "transactionsData" td ON "tr"."counterpartyBeneficiaryId" = td."counterpartyBeneficiaryId" @@ -789,6 +790,8 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti excludePaymentMethods, transactionType = [], + + subjectColumn, }: DailySingleTransactionAmountType) { if (!projectId) { throw new Error('projectId is required'); @@ -828,16 +831,18 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti if (ruleType === 'amount') { conditions.push(Prisma.sql`"transactionBaseAmount" > ${amountThreshold}`); - query = Prisma.sql`SELECT "counterpartyBeneficiaryId" AS "counterpartyId" FROM "TransactionRecord" "tr" WHERE ${Prisma.join( + query = Prisma.sql`SELECT ${Prisma.raw( + subjectColumn, + )} FROM "TransactionRecord" "tr" WHERE ${Prisma.join( conditions, ' AND ', - )} GROUP BY "counterpartyBeneficiaryId"`; + )} GROUP BY "${Prisma.raw(subjectColumn)}"`; } else if (ruleType === 'count') { - query = Prisma.sql`SELECT "counterpartyBeneficiaryId" as "counterpartyId", + query = Prisma.sql`SELECT ${Prisma.raw(subjectColumn)} COUNT(id) AS "transactionCount" FROM "TransactionRecord" "tr" WHERE ${Prisma.join( conditions, ' AND ', - )} GROUP BY "counterpartyBeneficiaryId" HAVING ${Prisma.raw( + )} GROUP BY ${Prisma.raw(subjectColumn)} HAVING ${Prisma.raw( `${AggregateType.COUNT}(id)`, )} > ${amountThreshold}`; } else { @@ -855,4 +860,142 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti return await this.prisma.$queryRaw(query); } + private buildTransactionsFiltersByAlert(inlineRule: InlineRule, alert?: Alert) { + const whereClause: Prisma.TransactionRecordWhereInput = {}; + + const filters: { + endDate: Date | undefined; + startDate: Date | undefined; + } = { + endDate: undefined, + startDate: undefined, + }; + + if (alert) { + const endDate = alert.updatedAt || alert.createdAt; + endDate.setHours(23, 59, 59, 999); + filters.endDate = endDate; + } + + // @ts-ignore - TODO: Replace logic with proper implementation for each rule + // eslint-disable-next-line + let { timeAmount, timeUnit } = inlineRule.options; + + if (!timeAmount || !timeUnit) { + if ( + inlineRule.fnName === 'evaluateHighVelocityHistoricAverage' && + inlineRule.options.lastDaysPeriod && + timeUnit + ) { + timeAmount = inlineRule.options.lastDaysPeriod.timeAmount; + } else { + return filters; + } + } + + let startDate = new Date(); + + let subtractValue = 0; + + const baseSubstractByMin = timeAmount * 60 * 1000; + + switch (timeUnit) { + case TIME_UNITS.minutes: + subtractValue = baseSubstractByMin; + break; + case TIME_UNITS.hours: + subtractValue = 60 * baseSubstractByMin; + break; + case TIME_UNITS.days: + subtractValue = 24 * 60 * baseSubstractByMin; + break; + case TIME_UNITS.months: + startDate.setMonth(startDate.getMonth() - timeAmount); + break; + case TIME_UNITS.years: + startDate.setFullYear(startDate.getFullYear() - timeAmount); + break; + } + + startDate.setHours(0, 0, 0, 0); + startDate = new Date(startDate.getTime() - subtractValue); + + if (filters.endDate) { + startDate = new Date(Math.min(startDate.getTime(), filters.endDate.getTime())); + } + + filters.startDate = startDate; + + if (filters.startDate) { + whereClause.transactionDate = { + gte: filters.startDate, + }; + } + + if (filters.endDate) { + whereClause.transactionDate = { + lte: filters.endDate, + }; + } + + return whereClause; + } + + getInvestigationFilter(projectId: string, inlineRule: InlineRule, subject: SubjectRecord) { + let investigationFilter; + + switch (inlineRule.fnInvestigationName) { + case 'investigateTransactionsAgainstDynamicRules': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + } + + if (!investigationFilter) { + this.logger.error(`No evaluation function found`, { + inlineRule, + }); + + throw new Error( + `No evaluation function found for rule name: ${(inlineRule as InlineRule).id}`, + ); + } + + return { + counterpartyBeneficiaryId: subject.counterpartyBeneficiaryId, + counterpartyOriginatorId: subject.counterpartyOriginatorId, + ...investigationFilter, + ...this.buildTransactionsFiltersByAlert(inlineRule), + projectId, + } satisfies Prisma.TransactionRecordWhereInput; + } + + investigateTransactionsAgainstDynamicRules(options: TransactionsAgainstDynamicRulesType) { + const { + amountBetween, + direction, + transactionType: _transactionType, + paymentMethods = [], + excludePaymentMethods = false, + projectId, + } = options; + + return { + projectId, + transactionAmount: { + gte: amountBetween?.min, + lte: amountBetween?.max, + }, + transactionDirection: direction, + transactionType: { + in: _transactionType as TransactionRecordType[], + }, + paymentMethod: { + ...(excludePaymentMethods + ? { notIn: paymentMethods as PaymentMethod[] } + : { in: paymentMethods as PaymentMethod[] }), + }, + } as const satisfies Prisma.TransactionRecordWhereInput; + } } diff --git a/services/workflows-service/src/data-analytics/types.ts b/services/workflows-service/src/data-analytics/types.ts index 727eaea79e..f084918178 100644 --- a/services/workflows-service/src/data-analytics/types.ts +++ b/services/workflows-service/src/data-analytics/types.ts @@ -1,53 +1,66 @@ import { TProjectId } from '@/types'; import { TransactionDirection, PaymentMethod, TransactionRecordType } from '@prisma/client'; import { AggregateType, TIME_UNITS } from './consts'; +import { Subject } from '@/alert/types'; export type InlineRule = { id: string; - subjects: string[] | readonly string[]; + // TODO: Keep only Subject type + subjects: ((Subject[] | readonly Subject[]) & string[]) | readonly string[]; } & ( | { fnName: 'evaluateHighTransactionTypePercentage'; + fnInvestigationName?: 'investigateHighTransactionTypePercentage'; options: Omit; } | { fnName: 'evaluateTransactionsAgainstDynamicRules'; + fnInvestigationName: 'investigateTransactionsAgainstDynamicRules'; options: Omit; } | { fnName: 'evaluateCustomersTransactionType'; + fnInvestigationName?: 'investigateCustomersTransactionType'; options: Omit; } | { fnName: 'evaluateTransactionAvg'; + fnInvestigationName?: 'investigateTransactionAvg'; options: Omit; } | { fnName: 'evaluateTransactionAvg'; + fnInvestigationName?: 'investigateTransactionAvg'; options: Omit; } | { fnName: 'evaluateDormantAccount'; + fnInvestigationName?: 'investigateDormantAccount'; options: Omit; } | { fnName: 'checkMerchantOngoingAlert'; + fnInvestigationName?: 'investigateMerchantOngoingAlert'; options: CheckRiskScoreOptions; } | { fnName: 'evaluateHighVelocityHistoricAverage'; + fnInvestigationName?: 'investigateHighVelocityHistoricAverage'; options: Omit; } | { fnName: 'evaluateMultipleMerchantsOneCounterparty'; + fnInvestigationName?: 'investigateMultipleMerchantsOneCounterparty'; options: Omit; } | { fnName: 'evaluateMerchantGroupAverage'; + fnInvestigationName?: 'investigateMerchantGroupAverage'; options: Omit; } | { fnName: 'evaluateDailySingleTransactionAmount'; + fnInvestigationName?: 'investigateDailySingleTransactionAmount'; options: Omit; } ); @@ -80,7 +93,7 @@ export type TransactionsAgainstDynamicRulesType = { export type HighTransactionTypePercentage = { projectId: TProjectId; transactionType: TransactionRecordType; - subjectColumn: 'counterpartyOriginatorId' | 'counterpartyBeneficiaryId'; + subjectColumn: Subject; minimumCount: number; minimumPercentage: number; timeAmount: number; @@ -184,5 +197,5 @@ export type DailySingleTransactionAmountType = { paymentMethods: PaymentMethod[] | readonly PaymentMethod[]; excludePaymentMethods: boolean; - // subjectColumn: 'counterpartyOriginatorId' | 'counterpartyBeneficiaryId'; + subjectColumn: Subject; }; diff --git a/services/workflows-service/src/global.d.ts b/services/workflows-service/src/global.d.ts index e1740dc74c..9ff5306786 100644 --- a/services/workflows-service/src/global.d.ts +++ b/services/workflows-service/src/global.d.ts @@ -4,6 +4,7 @@ declare module '@prisma/client' { WorkflowDefinition as _WorkflowDefinition, Alert as _Alert, } from '@prisma/client/index'; + import { TExecutionDetails } from '@/alert/types'; import type { WorkflowConfig } from '@/workflow/schemas/zod-schemas'; import type { TWorkflowExtenstion } from '@/workflow/schemas/extenstions.schemas'; import type { TCustomerConfig, TCustomerSubscription } from '@/customer/schemas/zod-schemas'; @@ -28,6 +29,6 @@ declare module '@prisma/client' { }; export type Alert = Omit<_Alert, 'executionDetails'> & { - executionDetails: TCustomerSubscription | any; + executionDetails: TCustomerSubscription | TExecutionDetails | any; }; } diff --git a/services/workflows-service/src/project/project-scope.service.ts b/services/workflows-service/src/project/project-scope.service.ts index e058edad69..2549e5eb9d 100644 --- a/services/workflows-service/src/project/project-scope.service.ts +++ b/services/workflows-service/src/project/project-scope.service.ts @@ -1,6 +1,9 @@ +import { logger } from '@ballerine/workflow-core'; import { Prisma } from '@prisma/client'; import type { TProjectIds } from '@/types'; import { Injectable } from '@nestjs/common'; +import { AppLoggerService } from '@/common/app-logger/app-logger.service'; +import { SentryService } from '@/sentry/sentry.service'; export interface PrismaGeneralQueryArgs { select?: Record | null; @@ -25,19 +28,39 @@ export interface PrismaGeneralUpsertArgs extends PrismaGeneralQueryArgs { @Injectable() export class ProjectScopeService { + constructor( + protected readonly logger: AppLoggerService, + protected readonly sentry: SentryService, + ) {} + scopeFindMany( args?: Prisma.SelectSubset, projectIds?: TProjectIds, ): T { // @ts-expect-error - dynamically typed for all queries args ||= {}; + + if (!projectIds) { + logger.error('Project IDs are required to scope the query', { data: args }); + const error = new Error('Project ID is null, projectId required to scope the query'); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: TODO create related error type + error.data = args; + + this.sentry.captureException(error); + + throw error; + } + // @ts-expect-error - dynamically typed for all queries args!.where = { // @ts-expect-error - dynamically typed for all queries ...args?.where, - project: { - id: { in: projectIds }, - }, + project: + typeof projectIds === 'string' + ? { id: projectIds } // Single ID + : { id: { in: projectIds } }, // Array of IDs, }; return args!; diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index 8b7bb034a2..f48efaee02 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -1,18 +1,17 @@ -import * as swagger from '@nestjs/swagger'; -import { TransactionService } from '@/transaction/transaction.service'; +import { UseCustomerAuthGuard } from '@/common/decorators/use-customer-auth-guard.decorator'; import { TransactionCreateAltDto, TransactionCreateAltDtoWrapper, TransactionCreateDto, } from '@/transaction/dtos/transaction-create.dto'; -import { UseCustomerAuthGuard } from '@/common/decorators/use-customer-auth-guard.decorator'; +import { TransactionService } from '@/transaction/transaction.service'; +import * as swagger from '@nestjs/swagger'; -import * as types from '@/types'; import { PrismaService } from '@/prisma/prisma.service'; +import * as types from '@/types'; -import { CurrentProject } from '@/common/decorators/current-project.decorator'; import { AppLoggerService } from '@/common/app-logger/app-logger.service'; -import express from 'express'; +import { CurrentProject } from '@/common/decorators/current-project.decorator'; import { Body, Controller, @@ -23,21 +22,25 @@ import { Res, ValidationPipe, } from '@nestjs/common'; +import express from 'express'; +import { AlertService } from '@/alert/alert.service'; +import { BulkStatus, TExecutionDetails } from '@/alert/types'; +import { TIME_UNITS } from '@/data-analytics/consts'; +import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; +import { InlineRule } from '@/data-analytics/types'; +import * as errors from '@/errors'; +import { exceptionValidationFactory } from '@/errors'; +import { ProjectScopeService } from '@/project/project-scope.service'; +import { BulkTransactionsCreatedDto } from '@/transaction/dtos/bulk-transactions-created.dto'; import { GetTransactionsByAlertDto, GetTransactionsDto, } from '@/transaction/dtos/get-transactions.dto'; -import { PaymentMethod } from '@prisma/client'; -import { BulkTransactionsCreatedDto } from '@/transaction/dtos/bulk-transactions-created.dto'; import { TransactionCreatedDto } from '@/transaction/dtos/transaction-created.dto'; -import { BulkStatus } from '@/alert/types'; -import * as errors from '@/errors'; -import { exceptionValidationFactory } from '@/errors'; -import { TIME_UNITS } from '@/data-analytics/consts'; +import { PaymentMethod } from '@prisma/client'; +import { isEmpty } from 'lodash'; import { TransactionEntityMapper } from './transaction.mapper'; -import { ProjectScopeService } from '@/project/project-scope.service'; -import { AlertService } from '@/alert/alert.service'; @swagger.ApiBearerAuth() @swagger.ApiTags('Transactions') @@ -49,6 +52,7 @@ export class TransactionControllerExternal { protected readonly prisma: PrismaService, protected readonly logger: AppLoggerService, protected readonly alertService: AlertService, + protected readonly dataAnalyticsService: DataAnalyticsService, ) {} @Post() @@ -239,7 +243,7 @@ export class TransactionControllerExternal { @Query() getTransactionsParameters: GetTransactionsDto, @CurrentProject() projectId: types.TProjectId, ) { - return this.service.getTransactions(getTransactionsParameters, projectId, { + return this.service.getTransactionsV1(getTransactionsParameters, projectId, { include: { counterpartyBeneficiary: { select: { @@ -330,32 +334,35 @@ export class TransactionControllerExternal { required: true, }) async getTransactionsByAlert( - @Query() getTransactionsByAlertParameters: GetTransactionsByAlertDto, + @Query() filters: GetTransactionsByAlertDto, @CurrentProject() projectId: types.TProjectId, ) { - const alert = await this.alertService.getAlertWithDefinition( - getTransactionsByAlertParameters.alertId, - projectId, - ); + const alert = await this.alertService.getAlertWithDefinition(filters.alertId, projectId); if (!alert) { - throw new errors.NotFoundException( - `Alert with id ${getTransactionsByAlertParameters.alertId} not found`, - ); + throw new errors.NotFoundException(`Alert with id ${filters.alertId} not found`); } if (!alert.alertDefinition) { throw new errors.NotFoundException(`Alert definition not found for alert ${alert.id}`); } - const filters: GetTransactionsByAlertDto = { - ...getTransactionsByAlertParameters, - ...(!getTransactionsByAlertParameters.startDate && !getTransactionsByAlertParameters.endDate - ? this.alertService.buildTransactionsFiltersByAlert(alert) - : {}), - }; + // Backward compatability will be remove soon, + if (isEmpty((alert.executionDetails as TExecutionDetails).filters)) { + return this.getTransactionsByAlertV1({ filters, projectId }); + } - return this.service.getTransactions(filters, projectId, { + return this.getTransactionsByAlertV2({ filters, projectId, alert }); + } + + private getTransactionsByAlertV1({ + filters, + projectId, + }: { + filters: GetTransactionsByAlertDto; + projectId: string; + }) { + return this.service.getTransactionsV1(filters, projectId, { include: { counterpartyBeneficiary: { select: { @@ -399,4 +406,69 @@ export class TransactionControllerExternal { }, }); } + + private getTransactionsByAlertV2({ + filters, + projectId, + alert, + }: { + filters: GetTransactionsByAlertDto; + projectId: string; + alert: Awaited>; + }) { + if (alert) { + return this.service.getTransactions(projectId, { + where: + alert.executionDetails.filters || + this.dataAnalyticsService.getInvestigationFilter( + projectId, + alert.alertDefinition.inlineRule as InlineRule, + alert.executionDetails.subjects, + ), + include: { + counterpartyBeneficiary: { + select: { + correlationId: true, + business: { + select: { + correlationId: true, + companyName: true, + }, + }, + endUser: { + select: { + correlationId: true, + firstName: true, + lastName: true, + }, + }, + }, + }, + counterpartyOriginator: { + select: { + correlationId: true, + business: { + select: { + correlationId: true, + companyName: true, + }, + }, + endUser: { + select: { + correlationId: true, + firstName: true, + lastName: true, + }, + }, + }, + }, + }, + orderBy: { + createdAt: 'desc', + }, + }); + } + + return []; + } } diff --git a/services/workflows-service/src/transaction/transaction.repository.ts b/services/workflows-service/src/transaction/transaction.repository.ts index 90d9be5a73..500e148821 100644 --- a/services/workflows-service/src/transaction/transaction.repository.ts +++ b/services/workflows-service/src/transaction/transaction.repository.ts @@ -6,6 +6,11 @@ import { TProjectId } from '@/types'; import { GetTransactionsDto } from './dtos/get-transactions.dto'; import { DateTimeFilter } from '@/common/query-filters/date-time-filter'; import { toPrismaOrderByGeneric } from '@/workflow/utils/toPrismaOrderBy'; +import deepmerge from 'deepmerge'; + +const DEFAULT_TRANSACTION_ORDER = { + transactionDate: Prisma.SortOrder.desc, +}; @Injectable() export class TransactionRepository { @@ -21,20 +26,36 @@ export class TransactionRepository { } async findMany( - args: Prisma.SelectSubset, projectId: TProjectId, + args?: Prisma.SelectSubset, ) { return await this.prisma.transactionRecord.findMany( - this.scopeService.scopeFindMany(args, [projectId]), + deepmerge(args || {}, this.scopeService.scopeFindMany(args, [projectId])), ); } - async findManyWithFilters( - getTransactionsParameters: GetTransactionsDto, - projectId: string, - options?: Prisma.TransactionRecordFindManyArgs, - ): Promise { - const args: Prisma.TransactionRecordFindManyArgs = {}; + // eslint-disable-next-line ballerine/verify-repository-project-scoped + static buildTransactionOrderByArgs(getTransactionsParameters: GetTransactionsDto) { + const args: { + orderBy: Prisma.TransactionRecordFindManyArgs['orderBy']; + } = { + orderBy: getTransactionsParameters.orderBy + ? toPrismaOrderByGeneric(getTransactionsParameters.orderBy) + : DEFAULT_TRANSACTION_ORDER, + }; + + return args; + } + + // eslint-disable-next-line ballerine/verify-repository-project-scoped + static buildTransactionPaginationArgs(getTransactionsParameters: GetTransactionsDto) { + const args: { + skip: Prisma.TransactionRecordFindManyArgs['skip']; + take?: Prisma.TransactionRecordFindManyArgs['take']; + } = { + take: 20, + skip: 0, + }; if (getTransactionsParameters.page?.number && getTransactionsParameters.page?.size) { // Temporary fix for pagination (class transformer issue) @@ -45,16 +66,25 @@ export class TransactionRepository { args.skip = size * (number - 1); } - if (getTransactionsParameters.orderBy) { - args.orderBy = toPrismaOrderByGeneric(getTransactionsParameters.orderBy); - } + return args; + } + + async findManyWithFilters( + getTransactionsParameters: GetTransactionsDto, + projectId: string, + options?: Prisma.TransactionRecordFindManyArgs, + ): Promise { + const args: Prisma.TransactionRecordFindManyArgs = { + ...TransactionRepository.buildTransactionPaginationArgs(getTransactionsParameters), + ...TransactionRepository.buildTransactionOrderByArgs(getTransactionsParameters), + }; return this.prisma.transactionRecord.findMany( this.scopeService.scopeFindMany( { ...options, where: { - ...this.buildFilters(getTransactionsParameters), + ...this.buildFiltersV1(getTransactionsParameters), }, ...args, }, @@ -64,7 +94,7 @@ export class TransactionRepository { } // eslint-disable-next-line ballerine/verify-repository-project-scoped - private buildFilters( + buildFiltersV1( getTransactionsParameters: GetTransactionsDto, ): Prisma.TransactionRecordWhereInput { const whereClause: Prisma.TransactionRecordWhereInput = {}; @@ -96,4 +126,24 @@ export class TransactionRepository { return whereClause; } + + // eslint-disable-next-line ballerine/verify-repository-project-scoped + buildFiltersV2( + projectId: TProjectId, + getTransactionsParameters: GetTransactionsDto, + args: Prisma.TransactionRecordWhereInput, + ): Prisma.TransactionRecordWhereInput { + const whereClause: Prisma.TransactionRecordWhereInput = { + productId: projectId, + transactionDate: { + gte: getTransactionsParameters.startDate, + lte: getTransactionsParameters.endDate, + }, + paymentMethod: getTransactionsParameters.paymentMethod, + }; + + deepmerge(args, whereClause); + + return whereClause; + } } diff --git a/services/workflows-service/src/transaction/transaction.service.ts b/services/workflows-service/src/transaction/transaction.service.ts index be3e58e7a2..317a00fd2f 100644 --- a/services/workflows-service/src/transaction/transaction.service.ts +++ b/services/workflows-service/src/transaction/transaction.service.ts @@ -68,15 +68,15 @@ export class TransactionService { return response; } - async getAll(args: Parameters[0], projectId: string) { - return this.repository.findMany(args, projectId); - } - - async getTransactions( + async getTransactionsV1( filters: GetTransactionsDto, projectId: string, args?: Parameters[2], ) { return this.repository.findManyWithFilters(filters, projectId, args); } + + async getTransactions(projectId: string, args?: Parameters[1]) { + return this.repository.findMany(projectId, args); + } } From b71d5ae596e8b969a763f80bdbb8be8f00b8e553 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sat, 16 Nov 2024 21:56:23 +0200 Subject: [PATCH 02/18] chore: remove filters dto and allow filter by alert only --- .../src/transaction/transaction.controller.external.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index f48efaee02..06a6479af8 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -352,7 +352,7 @@ export class TransactionControllerExternal { return this.getTransactionsByAlertV1({ filters, projectId }); } - return this.getTransactionsByAlertV2({ filters, projectId, alert }); + return this.getTransactionsByAlertV2({ projectId, alert }); } private getTransactionsByAlertV1({ @@ -408,11 +408,9 @@ export class TransactionControllerExternal { } private getTransactionsByAlertV2({ - filters, projectId, alert, }: { - filters: GetTransactionsByAlertDto; projectId: string; alert: Awaited>; }) { From aade29f97940c73cf5912f70601ada3f04f87694 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 17 Nov 2024 00:10:15 +0200 Subject: [PATCH 03/18] chore: fixing filters --- services/workflows-service/src/app.module.ts | 5 ++++ .../data-analytics/data-analytics.service.ts | 29 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/services/workflows-service/src/app.module.ts b/services/workflows-service/src/app.module.ts index 5de43b47b1..b9966c8509 100644 --- a/services/workflows-service/src/app.module.ts +++ b/services/workflows-service/src/app.module.ts @@ -77,6 +77,11 @@ export const validate = async (config: Record) => { @Module({ controllers: [SwaggerController], imports: [ + ConfigModule.forRoot({ + isGlobal: true, + envFilePath: [`.env.${process.env.ENVIRONMENT_NAME}`, '.env'], + cache: true, + }), SentryModule, MulterModule.registerAsync({ imports: [ConfigModule], diff --git a/services/workflows-service/src/data-analytics/data-analytics.service.ts b/services/workflows-service/src/data-analytics/data-analytics.service.ts index 8d0adaae6e..57100a2fe4 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.service.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.service.ts @@ -963,8 +963,7 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti } return { - counterpartyBeneficiaryId: subject.counterpartyBeneficiaryId, - counterpartyOriginatorId: subject.counterpartyOriginatorId, + ...subject, ...investigationFilter, ...this.buildTransactionsFiltersByAlert(inlineRule), projectId, @@ -975,7 +974,7 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti const { amountBetween, direction, - transactionType: _transactionType, + transactionType, paymentMethods = [], excludePaymentMethods = false, projectId, @@ -983,14 +982,22 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti return { projectId, - transactionAmount: { - gte: amountBetween?.min, - lte: amountBetween?.max, - }, - transactionDirection: direction, - transactionType: { - in: _transactionType as TransactionRecordType[], - }, + ...(amountBetween + ? { + transactionAmount: { + gte: amountBetween?.min, + lte: amountBetween?.max, + }, + } + : {}), + ...(direction ? { transactionDirection: direction } : {}), + ...(transactionType + ? { + transactionType: { + in: transactionType as TransactionRecordType[], + }, + } + : {}), paymentMethod: { ...(excludePaymentMethods ? { notIn: paymentMethods as PaymentMethod[] } From cfac7f9909f4b6e9928f564c9189ece6b3b1ecac Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 17 Nov 2024 00:50:17 +0200 Subject: [PATCH 04/18] chore: reset scope service --- .../src/project/project-scope.service.ts | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/services/workflows-service/src/project/project-scope.service.ts b/services/workflows-service/src/project/project-scope.service.ts index 2549e5eb9d..e058edad69 100644 --- a/services/workflows-service/src/project/project-scope.service.ts +++ b/services/workflows-service/src/project/project-scope.service.ts @@ -1,9 +1,6 @@ -import { logger } from '@ballerine/workflow-core'; import { Prisma } from '@prisma/client'; import type { TProjectIds } from '@/types'; import { Injectable } from '@nestjs/common'; -import { AppLoggerService } from '@/common/app-logger/app-logger.service'; -import { SentryService } from '@/sentry/sentry.service'; export interface PrismaGeneralQueryArgs { select?: Record | null; @@ -28,39 +25,19 @@ export interface PrismaGeneralUpsertArgs extends PrismaGeneralQueryArgs { @Injectable() export class ProjectScopeService { - constructor( - protected readonly logger: AppLoggerService, - protected readonly sentry: SentryService, - ) {} - scopeFindMany( args?: Prisma.SelectSubset, projectIds?: TProjectIds, ): T { // @ts-expect-error - dynamically typed for all queries args ||= {}; - - if (!projectIds) { - logger.error('Project IDs are required to scope the query', { data: args }); - const error = new Error('Project ID is null, projectId required to scope the query'); - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore: TODO create related error type - error.data = args; - - this.sentry.captureException(error); - - throw error; - } - // @ts-expect-error - dynamically typed for all queries args!.where = { // @ts-expect-error - dynamically typed for all queries ...args?.where, - project: - typeof projectIds === 'string' - ? { id: projectIds } // Single ID - : { id: { in: projectIds } }, // Array of IDs, + project: { + id: { in: projectIds }, + }, }; return args!; From 10c0a8e54688f85e8d3e48cbe204a3377f2beb62 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Mon, 18 Nov 2024 23:04:35 +0200 Subject: [PATCH 05/18] feat: fix ui to load relevant counterparty --- .../src/domains/transactions/fetchers.ts | 2 +- .../useTransactionsQuery.tsx | 2 +- .../src/domains/transactions/query-keys.ts | 2 +- .../src/alert/alert.controller.external.ts | 62 ++++++++----------- services/workflows-service/src/alert/types.ts | 8 +++ .../src/project/project-scope.service.ts | 7 ++- .../transaction.controller.external.ts | 15 +++-- 7 files changed, 48 insertions(+), 50 deletions(-) diff --git a/apps/backoffice-v2/src/domains/transactions/fetchers.ts b/apps/backoffice-v2/src/domains/transactions/fetchers.ts index 4a2c7e13be..ae7c46a971 100644 --- a/apps/backoffice-v2/src/domains/transactions/fetchers.ts +++ b/apps/backoffice-v2/src/domains/transactions/fetchers.ts @@ -178,7 +178,7 @@ export const TransactionsListSchema = z.array( export type TTransactionsList = z.output; export const fetchTransactions = async (params: { - counterpartyId: string; + counterpartyId?: string; page: { number: number; size: number; diff --git a/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx b/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx index 1f43e0d085..ca7a384ce3 100644 --- a/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx +++ b/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx @@ -22,7 +22,7 @@ export const useTransactionsQuery = ({ page, pageSize, }), - enabled: isAuthenticated && !!counterpartyId, + enabled: isAuthenticated, staleTime: 100_000, }); }; diff --git a/apps/backoffice-v2/src/domains/transactions/query-keys.ts b/apps/backoffice-v2/src/domains/transactions/query-keys.ts index 158d98ec23..f806c25f79 100644 --- a/apps/backoffice-v2/src/domains/transactions/query-keys.ts +++ b/apps/backoffice-v2/src/domains/transactions/query-keys.ts @@ -8,7 +8,7 @@ export const transactionsQueryKeys = createQueryKeys('transactions', { ...params }: { alertId: string; - counterpartyId: string; + counterpartyId?: string; page: number; pageSize: number; }) => { diff --git a/services/workflows-service/src/alert/alert.controller.external.ts b/services/workflows-service/src/alert/alert.controller.external.ts index 6ba09567a9..6be0efbe27 100644 --- a/services/workflows-service/src/alert/alert.controller.external.ts +++ b/services/workflows-service/src/alert/alert.controller.external.ts @@ -116,47 +116,22 @@ export class AlertControllerExternal { }, }, }, - // TODO: remove this after migration - counterparty: { - select: { - id: true, - business: { - select: { - id: true, - correlationId: true, - companyName: true, - }, - }, - endUser: { - select: { - id: true, - correlationId: true, - firstName: true, - lastName: true, - }, - }, - }, - }, }, }, ); return alerts.map(alert => { - const { alertDefinition, assignee, counterparty, state, ...alertWithoutDefinition } = - alert as TAlertTransactionResponse; + const { + alertDefinition, + assignee, + counterpartyBeneficiary, + counterpartyOriginator, + state, + ...alertWithoutDefinition + } = alert as TAlertTransactionResponse; - return { - ...alertWithoutDefinition, - correlationId: alertDefinition.correlationId, - assignee: assignee - ? { - id: assignee?.id, - fullName: `${assignee?.firstName} ${assignee?.lastName}`, - avatarUrl: assignee?.avatarUrl, - } - : null, - alertDetails: alertDefinition.description, - subject: counterparty.business + const counterpartyDetails = (counterparty: TAlertTransactionResponse['counterparty']) => + counterparty.business ? { type: 'business', id: counterparty.business.id, @@ -168,7 +143,22 @@ export class AlertControllerExternal { id: counterparty.endUser.id, correlationId: counterparty.endUser.correlationId, name: `${counterparty.endUser.firstName} ${counterparty.endUser.lastName}`, - }, + }; + + return { + ...alertWithoutDefinition, + correlationId: alertDefinition.correlationId, + assignee: assignee + ? { + id: assignee?.id, + fullName: `${assignee?.firstName} ${assignee?.lastName}`, + avatarUrl: assignee?.avatarUrl, + } + : null, + alertDetails: alertDefinition.description, + subject: + counterpartyDetails(counterpartyBeneficiary) || + counterpartyDetails(counterpartyOriginator), decision: state, }; }); diff --git a/services/workflows-service/src/alert/types.ts b/services/workflows-service/src/alert/types.ts index 436130208f..a89966cf24 100644 --- a/services/workflows-service/src/alert/types.ts +++ b/services/workflows-service/src/alert/types.ts @@ -38,6 +38,14 @@ export type TAlertTransactionResponse = TAlertResponse & { business: Pick; endUser: Pick; }; + counterpartyBeneficiary: { + business: Pick; + endUser: Pick; + }; + counterpartyOriginator: { + business: Pick; + endUser: Pick; + }; }; export type TAlertMerchantResponse = TAlertResponse & { diff --git a/services/workflows-service/src/project/project-scope.service.ts b/services/workflows-service/src/project/project-scope.service.ts index e058edad69..31af0251a3 100644 --- a/services/workflows-service/src/project/project-scope.service.ts +++ b/services/workflows-service/src/project/project-scope.service.ts @@ -35,9 +35,10 @@ export class ProjectScopeService { args!.where = { // @ts-expect-error - dynamically typed for all queries ...args?.where, - project: { - id: { in: projectIds }, - }, + project: + typeof projectIds === 'string' + ? { id: projectIds } // Single ID + : { id: { in: projectIds } }, // Array of IDs }; return args!; diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index 06a6479af8..97dd6b6ad2 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -347,7 +347,7 @@ export class TransactionControllerExternal { throw new errors.NotFoundException(`Alert definition not found for alert ${alert.id}`); } - // Backward compatability will be remove soon, + // Backward compatibility will be remove soon, if (isEmpty((alert.executionDetails as TExecutionDetails).filters)) { return this.getTransactionsByAlertV1({ filters, projectId }); } @@ -416,13 +416,12 @@ export class TransactionControllerExternal { }) { if (alert) { return this.service.getTransactions(projectId, { - where: - alert.executionDetails.filters || - this.dataAnalyticsService.getInvestigationFilter( - projectId, - alert.alertDefinition.inlineRule as InlineRule, - alert.executionDetails.subjects, - ), + where: alert.executionDetails.filters, + // || this.dataAnalyticsService.getInvestigationFilter( + // projectId, + // alert.alertDefinition.inlineRule as InlineRule, + // alert.executionDetails.subjects, + // ), include: { counterpartyBeneficiary: { select: { From 9acec616b52ac3b46867ecb9edeabe899c1b8654 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Wed, 20 Nov 2024 16:10:24 +0200 Subject: [PATCH 06/18] fix: fetch all transaction and setup order from dto --- .../useTransactionsQuery.tsx | 2 +- ...ctionMonitoringAlertsAnalysisPageLogic.tsx | 7 ++-- .../src/alert/alert.repository.ts | 4 +- .../src/alert/alert.service.ts | 30 ++++++++++++-- services/workflows-service/src/alert/types.ts | 7 ++++ .../src/data-analytics/utils.ts | 23 +++++++++++ .../transaction.controller.external.ts | 18 ++------ .../src/transaction/transaction.repository.ts | 41 ++++++++++++++----- .../src/transaction/transaction.service.ts | 22 ++++++++-- 9 files changed, 115 insertions(+), 39 deletions(-) diff --git a/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx b/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx index ca7a384ce3..04e7b57478 100644 --- a/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx +++ b/apps/backoffice-v2/src/domains/transactions/hooks/queries/useTransactionsQuery/useTransactionsQuery.tsx @@ -9,7 +9,7 @@ export const useTransactionsQuery = ({ pageSize, }: { alertId: string; - counterpartyId: string; + counterpartyId?: string; page: number; pageSize: number; }) => { diff --git a/apps/backoffice-v2/src/pages/TransactionMonitoringAlertsAnalysis/hooks/useTransactionMonitoringAlertsAnalysisPageLogic/useTransactionMonitoringAlertsAnalysisPageLogic.tsx b/apps/backoffice-v2/src/pages/TransactionMonitoringAlertsAnalysis/hooks/useTransactionMonitoringAlertsAnalysisPageLogic/useTransactionMonitoringAlertsAnalysisPageLogic.tsx index 3eda80cb28..ee98601bfa 100644 --- a/apps/backoffice-v2/src/pages/TransactionMonitoringAlertsAnalysis/hooks/useTransactionMonitoringAlertsAnalysisPageLogic/useTransactionMonitoringAlertsAnalysisPageLogic.tsx +++ b/apps/backoffice-v2/src/pages/TransactionMonitoringAlertsAnalysis/hooks/useTransactionMonitoringAlertsAnalysisPageLogic/useTransactionMonitoringAlertsAnalysisPageLogic.tsx @@ -5,19 +5,18 @@ import { useTransactionsQuery } from '@/domains/transactions/hooks/queries/useTr import { useCallback } from 'react'; export const useTransactionMonitoringAlertsAnalysisPageLogic = () => { - const [{ businessId, counterpartyId }] = useSerializedSearchParams(); + const [{ counterpartyId }] = useSerializedSearchParams(); const { alertId } = useParams(); const { data: alertDefinition, isLoading: isLoadingAlertDefinition } = useAlertDefinitionByAlertIdQuery({ alertId: alertId ?? '', }); const { data: transactions } = useTransactionsQuery({ - alertId: alertId ?? '', - businessId: businessId ?? '', + alertId: alertId?.toString() ?? '', // @TODO: Remove counterpartyId: counterpartyId ?? '', page: 1, - pageSize: 50, + pageSize: 500, }); const navigate = useNavigate(); const onNavigateBack = useCallback(() => { diff --git a/services/workflows-service/src/alert/alert.repository.ts b/services/workflows-service/src/alert/alert.repository.ts index b059342d8d..31010f290e 100644 --- a/services/workflows-service/src/alert/alert.repository.ts +++ b/services/workflows-service/src/alert/alert.repository.ts @@ -17,8 +17,8 @@ export class AlertRepository { return await this.prisma.alert.create(args); } - async findFirst>( - args: Prisma.SelectSubset>, + async findFirst>( + args: Prisma.SelectSubset>, projectIds: TProjectIds, ) { const queryArgs = this.scopeService.scopeFindFirst(args, projectIds); diff --git a/services/workflows-service/src/alert/alert.service.ts b/services/workflows-service/src/alert/alert.service.ts index c255798928..7664659ac1 100644 --- a/services/workflows-service/src/alert/alert.service.ts +++ b/services/workflows-service/src/alert/alert.service.ts @@ -16,16 +16,20 @@ import { AlertState, AlertStatus, MonitoringType, - Prisma, } from '@prisma/client'; import _ from 'lodash'; import { AlertExecutionStatus } from './consts'; import { FindAlertsDto } from './dtos/get-alerts.dto'; -import { TDedupeStrategy, TExecutionDetails } from './types'; +import { DedupeWindow, TDedupeStrategy, TExecutionDetails } from './types'; import { computeHash } from '@ballerine/common'; +import { convertTimeUnitToMilliseconds } from '@/data-analytics/utils'; const DEFAULT_DEDUPE_STRATEGIES = { cooldownTimeframeInMinutes: 60 * 24, + dedupeWindow: { + timeAmount: 7, + timeUnit: TIME_UNITS.days, + }, }; @Injectable() @@ -383,13 +387,17 @@ export class AlertService { return true; } - const { cooldownTimeframeInMinutes } = dedupeStrategy || DEFAULT_DEDUPE_STRATEGIES; + const { cooldownTimeframeInMinutes, dedupeWindow } = + dedupeStrategy || DEFAULT_DEDUPE_STRATEGIES; const existingAlert = await this.alertRepository.findFirst( { where: { AND: [{ alertDefinitionId: alertDefinition.id }, ...subjectPayload], }, + orderBy: { + createdAt: 'desc', // Ensure we're getting the most recent alert + }, }, [alertDefinition.projectId], ); @@ -398,6 +406,10 @@ export class AlertService { return false; } + if (this._isTriggeredSinceLastDedupe(existingAlert, dedupeWindow)) { + return true; + } + const cooldownDurationInMs = cooldownTimeframeInMinutes * 60 * 1000; // Calculate the timestamp after which alerts will be considered outside the cooldown period @@ -418,6 +430,18 @@ export class AlertService { return false; } + private _isTriggeredSinceLastDedupe(existingAlert: Alert, dedupeWindow: DedupeWindow): boolean { + if (!existingAlert.dedupedAt || !dedupeWindow) { + return false; + } + + const dedupeWindowDurationInMs = convertTimeUnitToMilliseconds(dedupeWindow); + + const dedupeWindowEndTime = existingAlert.dedupedAt.getTime() + dedupeWindowDurationInMs; + + return Date.now() > dedupeWindowEndTime; + } + private getStatusFromState(newState: AlertState): ObjectValues { const alertStateToStatusMap = { [AlertState.triggered]: AlertStatus.new, diff --git a/services/workflows-service/src/alert/types.ts b/services/workflows-service/src/alert/types.ts index a89966cf24..e9eb202bc3 100644 --- a/services/workflows-service/src/alert/types.ts +++ b/services/workflows-service/src/alert/types.ts @@ -1,3 +1,4 @@ +import { TIME_UNITS } from '@/data-analytics/consts'; import { Alert, AlertDefinition, Business, EndUser, Prisma, User } from '@prisma/client'; // TODO: Remove counterpartyId from SubjectRecord @@ -19,6 +20,12 @@ export type TExecutionDetails = { export type TDedupeStrategy = { mute: boolean; cooldownTimeframeInMinutes: number; + dedupeWindow: DedupeWindow; +}; + +export type DedupeWindow = { + timeAmount: number; + timeUnit: (typeof TIME_UNITS)[keyof typeof TIME_UNITS]; }; export const BulkStatus = { diff --git a/services/workflows-service/src/data-analytics/utils.ts b/services/workflows-service/src/data-analytics/utils.ts index 16cb89e691..8eac88412e 100644 --- a/services/workflows-service/src/data-analytics/utils.ts +++ b/services/workflows-service/src/data-analytics/utils.ts @@ -30,3 +30,26 @@ export const calculateStartDate = (timeUnit: TimeUnit, timeAmount: number): Date return startDate; }; + +export const convertTimeUnitToMilliseconds = (dedupeWindow: { + timeAmount: number; + timeUnit: TimeUnit; +}): number => { + let multiplier = 0; + + switch (dedupeWindow.timeUnit) { + case 'days': + multiplier = 24 * 60 * 60 * 1000; // Convert days to milliseconds + break; + case 'hours': + multiplier = 60 * 60 * 1000; // Convert hours to milliseconds + break; + case 'minutes': + multiplier = 60 * 1000; // Convert minutes to milliseconds + break; + default: + throw new Error(`Unknown time unit: ${dedupeWindow.timeUnit}`); + } + + return dedupeWindow.timeAmount * multiplier; +}; diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index 97dd6b6ad2..19a7833159 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -28,7 +28,6 @@ import { AlertService } from '@/alert/alert.service'; import { BulkStatus, TExecutionDetails } from '@/alert/types'; import { TIME_UNITS } from '@/data-analytics/consts'; import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; -import { InlineRule } from '@/data-analytics/types'; import * as errors from '@/errors'; import { exceptionValidationFactory } from '@/errors'; import { ProjectScopeService } from '@/project/project-scope.service'; @@ -352,7 +351,7 @@ export class TransactionControllerExternal { return this.getTransactionsByAlertV1({ filters, projectId }); } - return this.getTransactionsByAlertV2({ projectId, alert }); + return this.getTransactionsByAlertV2({ projectId, alert, filters }); } private getTransactionsByAlertV1({ @@ -401,27 +400,21 @@ export class TransactionControllerExternal { }, }, }, - orderBy: { - createdAt: 'desc', - }, }); } private getTransactionsByAlertV2({ projectId, alert, + filters, }: { projectId: string; alert: Awaited>; + filters: Pick; }) { if (alert) { - return this.service.getTransactions(projectId, { + return this.service.getTransactions(projectId, filters, { where: alert.executionDetails.filters, - // || this.dataAnalyticsService.getInvestigationFilter( - // projectId, - // alert.alertDefinition.inlineRule as InlineRule, - // alert.executionDetails.subjects, - // ), include: { counterpartyBeneficiary: { select: { @@ -460,9 +453,6 @@ export class TransactionControllerExternal { }, }, }, - orderBy: { - createdAt: 'desc', - }, }); } diff --git a/services/workflows-service/src/transaction/transaction.repository.ts b/services/workflows-service/src/transaction/transaction.repository.ts index 500e148821..4ea0d0d72f 100644 --- a/services/workflows-service/src/transaction/transaction.repository.ts +++ b/services/workflows-service/src/transaction/transaction.repository.ts @@ -7,6 +7,7 @@ import { GetTransactionsDto } from './dtos/get-transactions.dto'; import { DateTimeFilter } from '@/common/query-filters/date-time-filter'; import { toPrismaOrderByGeneric } from '@/workflow/utils/toPrismaOrderBy'; import deepmerge from 'deepmerge'; +import { PageDto } from '@/common/dto'; const DEFAULT_TRANSACTION_ORDER = { transactionDate: Prisma.SortOrder.desc, @@ -35,11 +36,13 @@ export class TransactionRepository { } // eslint-disable-next-line ballerine/verify-repository-project-scoped - static buildTransactionOrderByArgs(getTransactionsParameters: GetTransactionsDto) { + static buildTransactionOrderByArgs( + getTransactionsParameters?: Pick, + ) { const args: { orderBy: Prisma.TransactionRecordFindManyArgs['orderBy']; } = { - orderBy: getTransactionsParameters.orderBy + orderBy: getTransactionsParameters?.orderBy ? toPrismaOrderByGeneric(getTransactionsParameters.orderBy) : DEFAULT_TRANSACTION_ORDER, }; @@ -48,7 +51,9 @@ export class TransactionRepository { } // eslint-disable-next-line ballerine/verify-repository-project-scoped - static buildTransactionPaginationArgs(getTransactionsParameters: GetTransactionsDto) { + static buildTransactionPaginationArgs( + getTransactionsParameters?: Pick, + ) { const args: { skip: Prisma.TransactionRecordFindManyArgs['skip']; take?: Prisma.TransactionRecordFindManyArgs['take']; @@ -57,7 +62,7 @@ export class TransactionRepository { skip: 0, }; - if (getTransactionsParameters.page?.number && getTransactionsParameters.page?.size) { + if (getTransactionsParameters?.page?.number && getTransactionsParameters.page?.size) { // Temporary fix for pagination (class transformer issue) const size = parseInt(getTransactionsParameters.page.size as unknown as string, 10); const number = parseInt(getTransactionsParameters.page.number as unknown as string, 10); @@ -69,7 +74,7 @@ export class TransactionRepository { return args; } - async findManyWithFilters( + async findManyWithFiltersV1( getTransactionsParameters: GetTransactionsDto, projectId: string, options?: Prisma.TransactionRecordFindManyArgs, @@ -93,6 +98,20 @@ export class TransactionRepository { ); } + async findManyWithFiltersV2( + getTransactionsParameters: GetTransactionsDto, + projectId: string, + options?: Prisma.TransactionRecordFindManyArgs, + ): Promise { + const args = deepmerge(options || {}, { + where: this.buildFiltersV2(getTransactionsParameters), + }); + + return this.prisma.transactionRecord.findMany( + this.scopeService.scopeFindMany(args, [projectId]), + ); + } + // eslint-disable-next-line ballerine/verify-repository-project-scoped buildFiltersV1( getTransactionsParameters: GetTransactionsDto, @@ -129,12 +148,14 @@ export class TransactionRepository { // eslint-disable-next-line ballerine/verify-repository-project-scoped buildFiltersV2( - projectId: TProjectId, getTransactionsParameters: GetTransactionsDto, - args: Prisma.TransactionRecordWhereInput, ): Prisma.TransactionRecordWhereInput { + const args: Prisma.TransactionRecordFindManyArgs = { + ...TransactionRepository.buildTransactionPaginationArgs(getTransactionsParameters), + ...TransactionRepository.buildTransactionOrderByArgs(getTransactionsParameters), + }; + const whereClause: Prisma.TransactionRecordWhereInput = { - productId: projectId, transactionDate: { gte: getTransactionsParameters.startDate, lte: getTransactionsParameters.endDate, @@ -142,8 +163,6 @@ export class TransactionRepository { paymentMethod: getTransactionsParameters.paymentMethod, }; - deepmerge(args, whereClause); - - return whereClause; + return deepmerge(args, whereClause); } } diff --git a/services/workflows-service/src/transaction/transaction.service.ts b/services/workflows-service/src/transaction/transaction.service.ts index 317a00fd2f..98470070e8 100644 --- a/services/workflows-service/src/transaction/transaction.service.ts +++ b/services/workflows-service/src/transaction/transaction.service.ts @@ -9,6 +9,8 @@ import { TransactionCreatedDto } from '@/transaction/dtos/transaction-created.dt import { SentryService } from '@/sentry/sentry.service'; import { isPrismaClientKnownRequestError } from '@/prisma/prisma.util'; import { getErrorMessageFromPrismaError } from '@/common/filters/HttpExceptions.filter'; +import { PageDto } from '@/common/dto'; +import { Prisma } from '@prisma/client'; @Injectable() export class TransactionService { @@ -71,12 +73,24 @@ export class TransactionService { async getTransactionsV1( filters: GetTransactionsDto, projectId: string, - args?: Parameters[2], + args?: Parameters[2], ) { - return this.repository.findManyWithFilters(filters, projectId, args); + return this.repository.findManyWithFiltersV1(filters, projectId, args); } - async getTransactions(projectId: string, args?: Parameters[1]) { - return this.repository.findMany(projectId, args); + async getTransactions( + projectId: string, + sortAndPageParams?: { + orderBy?: `${string}:asc` | `${string}:desc`; + page: PageDto; + }, + args?: Parameters[1], + ) { + const sortAndPageArgs: Prisma.TransactionRecordFindManyArgs = { + ...TransactionRepository.buildTransactionPaginationArgs(sortAndPageParams), + ...TransactionRepository.buildTransactionOrderByArgs(sortAndPageParams), + }; + + return this.repository.findMany(projectId, { ...args, ...sortAndPageArgs }); } } From f68631636a7a3e3e5f5f17062eb884b28fe43381 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Thu, 21 Nov 2024 08:19:04 +0200 Subject: [PATCH 07/18] fix: investigation data --- .../src/alert/alert.service.ts | 6 +- .../data-analytics/data-analytics.module.ts | 5 +- .../data-analytics/data-analytics.service.ts | 174 +-------- .../data-investigation.service.ts | 334 ++++++++++++++++++ .../src/transaction/transaction.module.ts | 5 +- 5 files changed, 357 insertions(+), 167 deletions(-) create mode 100644 services/workflows-service/src/data-analytics/data-investigation.service.ts diff --git a/services/workflows-service/src/alert/alert.service.ts b/services/workflows-service/src/alert/alert.service.ts index 7664659ac1..49ed9c1135 100644 --- a/services/workflows-service/src/alert/alert.service.ts +++ b/services/workflows-service/src/alert/alert.service.ts @@ -23,6 +23,7 @@ import { FindAlertsDto } from './dtos/get-alerts.dto'; import { DedupeWindow, TDedupeStrategy, TExecutionDetails } from './types'; import { computeHash } from '@ballerine/common'; import { convertTimeUnitToMilliseconds } from '@/data-analytics/utils'; +import { DataInvestigationService } from '@/data-analytics/data-investigation.service'; const DEFAULT_DEDUPE_STRATEGIES = { cooldownTimeframeInMinutes: 60 * 24, @@ -38,6 +39,7 @@ export class AlertService { private readonly prisma: PrismaService, private readonly logger: AppLoggerService, private readonly dataAnalyticsService: DataAnalyticsService, + private readonly dataInvestigationService: DataInvestigationService, private readonly alertRepository: AlertRepository, private readonly alertDefinitionRepository: AlertDefinitionRepository, ) {} @@ -358,7 +360,7 @@ export class AlertService { }, subject: mergedSubject, executionRow, - filters: this.dataAnalyticsService.getInvestigationFilter( + filters: this.dataInvestigationService.getInvestigationFilter( projectId, alertDef.inlineRule as InlineRule, mergedSubject, @@ -407,7 +409,7 @@ export class AlertService { } if (this._isTriggeredSinceLastDedupe(existingAlert, dedupeWindow)) { - return true; + return false; } const cooldownDurationInMs = cooldownTimeframeInMinutes * 60 * 1000; diff --git a/services/workflows-service/src/data-analytics/data-analytics.module.ts b/services/workflows-service/src/data-analytics/data-analytics.module.ts index 29cbcb5af4..0f5f5c7b8c 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.module.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.module.ts @@ -9,6 +9,7 @@ import { ProjectScopeService } from '@/project/project-scope.service'; import { BusinessReportModule } from '@/business-report/business-report.module'; // eslint-disable-next-line import/no-cycle import { AlertModule } from '@/alert/alert.module'; +import { DataInvestigationService } from './data-investigation.service'; @Module({ imports: [ @@ -18,7 +19,7 @@ import { AlertModule } from '@/alert/alert.module'; forwardRef(() => AlertModule), ], controllers: [DataAnalyticsControllerInternal, DataAnalyticsControllerExternal], - providers: [DataAnalyticsService, ProjectScopeService], - exports: [ACLModule, DataAnalyticsService], + providers: [DataAnalyticsService, ProjectScopeService, DataInvestigationService], + exports: [ACLModule, DataAnalyticsService, DataInvestigationService], }) export class DataAnalyticsModule {} diff --git a/services/workflows-service/src/data-analytics/data-analytics.service.ts b/services/workflows-service/src/data-analytics/data-analytics.service.ts index 57100a2fe4..537cbaff0a 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.service.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.service.ts @@ -1,26 +1,25 @@ -import { Injectable } from '@nestjs/common'; +import { MERCHANT_REPORT_TYPES_MAP, MerchantReportType } from '@/business-report/constants'; +import { AppLoggerService } from '@/common/app-logger/app-logger.service'; import { PrismaService } from '@/prisma/prisma.service'; +import { Injectable } from '@nestjs/common'; +import { AlertSeverity, Prisma } from '@prisma/client'; +import { isEmpty } from 'lodash'; +import { AggregateType } from './consts'; import { CheckRiskScoreOptions, + DailySingleTransactionAmountType, HighTransactionTypePercentage, HighVelocityHistoricAverageOptions, InlineRule, TCustomersTransactionTypeOptions, TDormantAccountOptions, - TPeerGroupTransactionAverageOptions, - TransactionsAgainstDynamicRulesType, - TMultipleMerchantsOneCounterparty, TExcludedCounterparty, TMerchantGroupAverage, - DailySingleTransactionAmountType, + TMultipleMerchantsOneCounterparty, + TPeerGroupTransactionAverageOptions, + TransactionsAgainstDynamicRulesType, } from './types'; -import { AggregateType, TIME_UNITS } from './consts'; import { calculateStartDate } from './utils'; -import { Alert, AlertSeverity, PaymentMethod, Prisma, TransactionRecordType } from '@prisma/client'; -import { AppLoggerService } from '@/common/app-logger/app-logger.service'; -import { isEmpty } from 'lodash'; -import { MERCHANT_REPORT_TYPES_MAP, MerchantReportType } from '@/business-report/constants'; -import { SubjectRecord } from '@/alert/types'; const COUNTERPARTY_ORIGINATOR_JOIN_CLAUSE = Prisma.sql`JOIN "Counterparty" AS "cpOriginator" ON "tr"."counterpartyOriginatorId" = "cpOriginator"."id"`; const COUNTERPARTY_BENEFICIARY_JOIN_CLAUSE = Prisma.sql`JOIN "Counterparty" AS "cpBeneficiary" ON "tr"."counterpartyBeneficiaryId" = "cpBeneficiary"."id"`; @@ -302,7 +301,7 @@ export class DataAnalyticsService { if (amountBetween) { conditions.push( - Prisma.sql`"transactionAmount" BETWEEN ${amountBetween.min} AND ${amountBetween.max}`, + Prisma.sql`"transactionBaseAmount" BETWEEN ${amountBetween.min} AND ${amountBetween.max}`, ); } @@ -389,7 +388,7 @@ export class DataAnalyticsService { "${Prisma.raw(subjectColumn)}" ) SELECT - "${Prisma.raw(subjectColumn)}" AS "counterpartyId" + "${Prisma.raw(subjectColumn)}" FROM "transactionsData" WHERE @@ -411,10 +410,10 @@ export class DataAnalyticsService { const conditions: Prisma.Sql[] = [ Prisma.sql`"tr"."projectId" = ${projectId}`, Prisma.sql`jsonb_exists(config, 'customer_expected_amount') AND ((config ->> 'customer_expected_amount')::numeric * ${factor}) != ${customerExpectedAmount}`, - Prisma.sql`"tr"."transactionAmount" > (config ->> 'customer_expected_amount')::numeric`, + Prisma.sql`"tr"."transactionBaseAmount" > (config ->> 'customer_expected_amount')::numeric`, ]; - const query: Prisma.Sql = Prisma.sql`SELECT "tr"."businessId" , "tr"."transactionAmount" FROM "TransactionRecord" as "tr" + const query: Prisma.Sql = Prisma.sql`SELECT "tr"."businessId" , "tr"."transactionBaseAmount" FROM "TransactionRecord" as "tr" WHERE ${Prisma.join(conditions, ' AND ')} `; const results = await this.prisma.$queryRaw(query); @@ -860,149 +859,4 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti return await this.prisma.$queryRaw(query); } - private buildTransactionsFiltersByAlert(inlineRule: InlineRule, alert?: Alert) { - const whereClause: Prisma.TransactionRecordWhereInput = {}; - - const filters: { - endDate: Date | undefined; - startDate: Date | undefined; - } = { - endDate: undefined, - startDate: undefined, - }; - - if (alert) { - const endDate = alert.updatedAt || alert.createdAt; - endDate.setHours(23, 59, 59, 999); - filters.endDate = endDate; - } - - // @ts-ignore - TODO: Replace logic with proper implementation for each rule - // eslint-disable-next-line - let { timeAmount, timeUnit } = inlineRule.options; - - if (!timeAmount || !timeUnit) { - if ( - inlineRule.fnName === 'evaluateHighVelocityHistoricAverage' && - inlineRule.options.lastDaysPeriod && - timeUnit - ) { - timeAmount = inlineRule.options.lastDaysPeriod.timeAmount; - } else { - return filters; - } - } - - let startDate = new Date(); - - let subtractValue = 0; - - const baseSubstractByMin = timeAmount * 60 * 1000; - - switch (timeUnit) { - case TIME_UNITS.minutes: - subtractValue = baseSubstractByMin; - break; - case TIME_UNITS.hours: - subtractValue = 60 * baseSubstractByMin; - break; - case TIME_UNITS.days: - subtractValue = 24 * 60 * baseSubstractByMin; - break; - case TIME_UNITS.months: - startDate.setMonth(startDate.getMonth() - timeAmount); - break; - case TIME_UNITS.years: - startDate.setFullYear(startDate.getFullYear() - timeAmount); - break; - } - - startDate.setHours(0, 0, 0, 0); - startDate = new Date(startDate.getTime() - subtractValue); - - if (filters.endDate) { - startDate = new Date(Math.min(startDate.getTime(), filters.endDate.getTime())); - } - - filters.startDate = startDate; - - if (filters.startDate) { - whereClause.transactionDate = { - gte: filters.startDate, - }; - } - - if (filters.endDate) { - whereClause.transactionDate = { - lte: filters.endDate, - }; - } - - return whereClause; - } - - getInvestigationFilter(projectId: string, inlineRule: InlineRule, subject: SubjectRecord) { - let investigationFilter; - - switch (inlineRule.fnInvestigationName) { - case 'investigateTransactionsAgainstDynamicRules': - investigationFilter = this[inlineRule.fnInvestigationName]({ - ...inlineRule.options, - projectId, - }); - } - - if (!investigationFilter) { - this.logger.error(`No evaluation function found`, { - inlineRule, - }); - - throw new Error( - `No evaluation function found for rule name: ${(inlineRule as InlineRule).id}`, - ); - } - - return { - ...subject, - ...investigationFilter, - ...this.buildTransactionsFiltersByAlert(inlineRule), - projectId, - } satisfies Prisma.TransactionRecordWhereInput; - } - - investigateTransactionsAgainstDynamicRules(options: TransactionsAgainstDynamicRulesType) { - const { - amountBetween, - direction, - transactionType, - paymentMethods = [], - excludePaymentMethods = false, - projectId, - } = options; - - return { - projectId, - ...(amountBetween - ? { - transactionAmount: { - gte: amountBetween?.min, - lte: amountBetween?.max, - }, - } - : {}), - ...(direction ? { transactionDirection: direction } : {}), - ...(transactionType - ? { - transactionType: { - in: transactionType as TransactionRecordType[], - }, - } - : {}), - paymentMethod: { - ...(excludePaymentMethods - ? { notIn: paymentMethods as PaymentMethod[] } - : { in: paymentMethods as PaymentMethod[] }), - }, - } as const satisfies Prisma.TransactionRecordWhereInput; - } } diff --git a/services/workflows-service/src/data-analytics/data-investigation.service.ts b/services/workflows-service/src/data-analytics/data-investigation.service.ts new file mode 100644 index 0000000000..2e57a53cb6 --- /dev/null +++ b/services/workflows-service/src/data-analytics/data-investigation.service.ts @@ -0,0 +1,334 @@ +import { SubjectRecord } from '@/alert/types'; +import { AppLoggerService } from '@/common/app-logger/app-logger.service'; +import { Injectable } from '@nestjs/common'; +import { Alert, PaymentMethod, Prisma, TransactionRecordType } from '@prisma/client'; +import { TIME_UNITS } from './consts'; +import { + DailySingleTransactionAmountType, + HighTransactionTypePercentage, + InlineRule, + TCustomersTransactionTypeOptions, + TDormantAccountOptions, + TMerchantGroupAverage, + TMultipleMerchantsOneCounterparty, + TPeerGroupTransactionAverageOptions, + TransactionsAgainstDynamicRulesType, +} from './types'; + +@Injectable() +export class DataInvestigationService { + constructor(protected readonly logger: AppLoggerService) {} + + getInvestigationFilter(projectId: string, inlineRule: InlineRule, subject: SubjectRecord) { + let investigationFilter; + + switch (inlineRule.fnInvestigationName) { + case 'investigateTransactionsAgainstDynamicRules': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateHighTransactionTypePercentage': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateDormantAccount': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateCustomersTransactionType': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateTransactionAvg': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateMultipleMerchantsOneCounterparty': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateMerchantGroupAverage': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + case 'investigateDailySingleTransactionAmount': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; + } + + if (!investigationFilter) { + this.logger.error(`No evaluation function found`, { + inlineRule, + }); + + throw new Error( + `No evaluation function found for rule name: ${(inlineRule as InlineRule).id}`, + ); + } + + return { + ...subject, + ...investigationFilter, + ...this._buildTransactionsFiltersByAlert(inlineRule), + projectId, + } satisfies Prisma.TransactionRecordWhereInput; + } + + investigateTransactionsAgainstDynamicRules(options: TransactionsAgainstDynamicRulesType) { + const { + amountBetween, + direction, + transactionType, + paymentMethods = [], + excludePaymentMethods = false, + projectId, + amountThreshold, + } = options; + + return { + projectId, + ...(amountBetween + ? { + transactionBaseAmount: { + gte: amountBetween?.min, + lte: amountBetween?.max, + }, + } + : {}), + ...(amountThreshold + ? { + transactionBaseAmount: { + gte: amountThreshold, + }, + } + : {}), + ...(direction ? { transactionDirection: direction } : {}), + ...(transactionType + ? { + transactionType: { + in: transactionType as TransactionRecordType[], + }, + } + : {}), + paymentMethod: { + ...(excludePaymentMethods + ? { notIn: paymentMethods as PaymentMethod[] } + : { in: paymentMethods as PaymentMethod[] }), + }, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateHighTransactionTypePercentage(options: HighTransactionTypePercentage) { + const { projectId, transactionType } = options; + + return { + projectId, + ...(transactionType + ? { + transactionType: transactionType, + } + : {}), + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateDormantAccount(options: TDormantAccountOptions) { + const { projectId } = options; + + return { + projectId, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateCustomersTransactionType(options: TCustomersTransactionTypeOptions) { + const { projectId, transactionType = [], paymentMethods = [] } = options; + + return { + projectId, + ...(paymentMethods + ? { + paymentMethod: { + in: paymentMethods as PaymentMethod[], + }, + } + : {}), + transactionType: { + in: transactionType as TransactionRecordType[], + }, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateTransactionAvg(options: TPeerGroupTransactionAverageOptions) { + const { projectId, transactionDirection, paymentMethod, minimumTransactionAmount } = options; + + return { + projectId, + paymentMethod: + paymentMethod.operator === '=' + ? { equals: paymentMethod.value } + : { not: paymentMethod.value }, + transactionBaseAmount: { + gte: minimumTransactionAmount, + }, + ...(transactionDirection ? { transactionDirection: transactionDirection } : {}), + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateMultipleMerchantsOneCounterparty(options: TMultipleMerchantsOneCounterparty) { + const { projectId } = options; + + return { + projectId, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateMerchantGroupAverage(options: TMerchantGroupAverage) { + const { projectId, paymentMethod } = options; + + return { + projectId, + paymentMethod: + paymentMethod.operator === '=' + ? { equals: paymentMethod.value } + : { not: paymentMethod.value }, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + investigateDailySingleTransactionAmount(options: DailySingleTransactionAmountType) { + const { + projectId, + + ruleType, + amountThreshold, + + direction, + + paymentMethods, + excludePaymentMethods, + + transactionType = [], + } = options; + + return { + projectId, + ...(amountThreshold + ? { + transactionBaseAmount: { + gte: amountThreshold, + }, + } + : {}), + ...(direction ? { transactionDirection: direction } : {}), + ...(transactionType + ? { + transactionType: { + in: transactionType as TransactionRecordType[], + }, + } + : {}), + paymentMethod: { + ...(excludePaymentMethods + ? { notIn: paymentMethods as PaymentMethod[] } + : { in: paymentMethods as PaymentMethod[] }), + }, + ...(ruleType === 'amount' ? { transactionBaseAmount: amountThreshold } : {}), + } as const satisfies Prisma.TransactionRecordWhereInput; + } + + _buildTransactionsFiltersByAlert(inlineRule: InlineRule, alert?: Alert) { + const whereClause: Prisma.TransactionRecordWhereInput = {}; + + const filters: { + endDate: Date | undefined; + startDate: Date | undefined; + } = { + endDate: undefined, + startDate: undefined, + }; + + if (alert) { + const endDate = alert.updatedAt || alert.createdAt; + endDate.setHours(23, 59, 59, 999); + filters.endDate = endDate; + } + + // @ts-ignore - TODO: Replace logic with proper implementation for each rule + // eslint-disable-next-line + let { timeAmount, timeUnit } = inlineRule.options; + + if (!timeAmount || !timeUnit) { + if ( + inlineRule.fnName === 'evaluateHighVelocityHistoricAverage' && + inlineRule.options.lastDaysPeriod && + timeUnit + ) { + timeAmount = inlineRule.options.lastDaysPeriod.timeAmount; + } else { + return filters; + } + } + + let startDate = new Date(); + + let subtractValue = 0; + + const baseSubstractByMin = timeAmount * 60 * 1000; + + switch (timeUnit) { + case TIME_UNITS.minutes: + subtractValue = baseSubstractByMin; + break; + case TIME_UNITS.hours: + subtractValue = 60 * baseSubstractByMin; + break; + case TIME_UNITS.days: + subtractValue = 24 * 60 * baseSubstractByMin; + break; + case TIME_UNITS.months: + startDate.setMonth(startDate.getMonth() - timeAmount); + break; + case TIME_UNITS.years: + startDate.setFullYear(startDate.getFullYear() - timeAmount); + break; + } + + startDate.setHours(0, 0, 0, 0); + startDate = new Date(startDate.getTime() - subtractValue); + + if (filters.endDate) { + startDate = new Date(Math.min(startDate.getTime(), filters.endDate.getTime())); + } + + filters.startDate = startDate; + + if (filters.startDate) { + whereClause.transactionDate = { + gte: filters.startDate, + }; + } + + if (filters.endDate) { + whereClause.transactionDate = { + lte: filters.endDate, + }; + } + + return whereClause; + } +} diff --git a/services/workflows-service/src/transaction/transaction.module.ts b/services/workflows-service/src/transaction/transaction.module.ts index 36e7865d17..afb8ff3996 100644 --- a/services/workflows-service/src/transaction/transaction.module.ts +++ b/services/workflows-service/src/transaction/transaction.module.ts @@ -8,12 +8,12 @@ import { PrismaModule } from '@/prisma/prisma.module'; import { ProjectScopeService } from '@/project/project-scope.service'; import { SentryService } from '@/sentry/sentry.service'; import { AlertService } from '@/alert/alert.service'; -import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; import { AlertRepository } from '@/alert/alert.repository'; import { AlertDefinitionRepository } from '@/alert-definition/alert-definition.repository'; +import { DataAnalyticsModule } from '@/data-analytics/data-analytics.module'; @Module({ - imports: [ACLModule, PrismaModule], + imports: [ACLModule, PrismaModule, DataAnalyticsModule], controllers: [TransactionControllerInternal, TransactionControllerExternal], providers: [ TransactionService, @@ -21,7 +21,6 @@ import { AlertDefinitionRepository } from '@/alert-definition/alert-definition.r ProjectScopeService, SentryService, AlertService, - DataAnalyticsService, AlertRepository, AlertDefinitionRepository, ], From 69d1a861d908b6e9d4255270023746bd7bab8947 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 24 Nov 2024 00:04:05 +0200 Subject: [PATCH 08/18] chore: fix tests and all around --- .../scripts/alerts/generate-alerts.ts | 33 ++++---- .../src/alert/alert.service.intg.test.ts | 83 +++++++++++-------- .../data-analytics/data-analytics.service.ts | 24 +++--- .../data-investigation.service.ts | 31 ++++++- .../src/data-analytics/types.ts | 20 ++--- 5 files changed, 116 insertions(+), 75 deletions(-) diff --git a/services/workflows-service/scripts/alerts/generate-alerts.ts b/services/workflows-service/scripts/alerts/generate-alerts.ts index ecfd4bdfaa..0b98f39114 100644 --- a/services/workflows-service/scripts/alerts/generate-alerts.ts +++ b/services/workflows-service/scripts/alerts/generate-alerts.ts @@ -382,7 +382,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HPC', fnName: 'evaluateHighTransactionTypePercentage', - fnInvestigationName: undefined, + fnInvestigationName: 'investigateHighTransactionTypePercentage', subjects: ['counterpartyOriginatorId'], options: { transactionType: TransactionRecordType.chargeback, @@ -401,6 +401,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'TLHAICC', fnName: 'evaluateTransactionAvg', + fnInvestigationName: 'investigateTransactionAvg', subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, @@ -424,6 +425,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'TLHAIAPM', fnName: 'evaluateTransactionAvg', + fnInvestigationName: 'investigateTransactionAvg', subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, @@ -447,6 +449,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PGAICT', fnName: 'evaluateTransactionAvg', + fnInvestigationName: 'investigateTransactionAvg', subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, @@ -471,6 +474,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'PGAIAPM', fnName: 'evaluateTransactionAvg', + fnInvestigationName: 'investigateTransactionAvg', subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, @@ -495,7 +499,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'DORMANT', fnName: 'evaluateDormantAccount', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateDormantAccount', + subjects: ['counterpartyBeneficiaryId'], options: { timeAmount: 180, timeUnit: TIME_UNITS.days, @@ -509,7 +514,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HVHAI_CC', fnName: 'evaluateHighVelocityHistoricAverage', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateHighVelocityHistoricAverage', + subjects: ['counterpartyBeneficiaryId'], options: { transactionDirection: TransactionDirection.inbound, minimumCount: 3, @@ -535,6 +541,7 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'HVHAI_APM', fnName: 'evaluateHighVelocityHistoricAverage', + fnInvestigationName: 'investigateHighVelocityHistoricAverage', subjects: ['counterpartyBeneficiaryId'], options: { minimumCount: 3, @@ -561,7 +568,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'MMOC_CC', fnName: 'evaluateMultipleMerchantsOneCounterparty', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateMultipleMerchantsOneCounterparty', + subjects: ['counterpartyOriginatorId'], options: { excludedCounterparty: { counterpartyBeneficiaryIds: ['9999999999999999', '999999______9999'], @@ -580,7 +588,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'MMOC_APM', fnName: 'evaluateMultipleMerchantsOneCounterparty', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateMultipleMerchantsOneCounterparty', + subjects: ['counterpartyOriginatorId'], options: { excludedCounterparty: { counterpartyBeneficiaryIds: ['9999999999999999', '999999______9999'], @@ -599,7 +608,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'MGAV_CC', fnName: 'evaluateMerchantGroupAverage', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateMerchantGroupAverage', + subjects: ['counterpartyBeneficiaryId'], options: { paymentMethod: { value: PaymentMethod.credit_card, @@ -619,7 +629,8 @@ export const ALERT_DEFINITIONS = { inlineRule: { id: 'MGAV_APM', fnName: 'evaluateMerchantGroupAverage', - subjects: ['counterpartyId'], + fnInvestigationName: 'investigateMerchantGroupAverage', + subjects: ['counterpartyBeneficiaryId'], options: { paymentMethod: { value: PaymentMethod.credit_card, @@ -653,8 +664,6 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, - - subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -679,8 +688,6 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, - - subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -705,8 +712,6 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, - - subjectColumn: 'counterpartyBeneficiaryId', }, }, }, @@ -731,8 +736,6 @@ export const ALERT_DEFINITIONS = { timeAmount: 1, timeUnit: TIME_UNITS.days, - - subjectColumn: 'counterpartyBeneficiaryId', }, }, }, diff --git a/services/workflows-service/src/alert/alert.service.intg.test.ts b/services/workflows-service/src/alert/alert.service.intg.test.ts index d757cf869c..99216e97c2 100644 --- a/services/workflows-service/src/alert/alert.service.intg.test.ts +++ b/services/workflows-service/src/alert/alert.service.intg.test.ts @@ -36,6 +36,8 @@ import { PrismaService } from '@/prisma/prisma.service'; import { BusinessService } from '@/business/business.service'; import { BusinessRepository } from '@/business/business.repository'; import { MerchantMonitoringClient } from '@/business-report/merchant-monitoring-client'; +import { DataInvestigationService } from '@/data-analytics/data-investigation.service'; +import { TIME_UNITS } from '@/data-analytics/consts'; type AsyncTransactionFactoryCallback = ( transactionFactory: TransactionFactory, @@ -92,6 +94,7 @@ describe('AlertService', () => { imports: commonTestingModules, providers: [ DataAnalyticsService, + DataInvestigationService, ProjectScopeService, AlertRepository, AlertDefinitionRepository, @@ -183,7 +186,7 @@ describe('AlertService', () => { }, ); - const counterpartyBeneficiary = + const counterpartyBeneficiaryId = baseTransactionFactory.data.counterpartyBeneficiary?.connect?.id; // Act @@ -193,7 +196,7 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counterpartyBeneficiary); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counterpartyBeneficiaryId); }); test('When there is no activity in the project', async () => { @@ -259,7 +262,9 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(transactions[0]?.counterpartyBeneficiaryId); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual( + transactions[0]?.counterpartyBeneficiaryId, + ); }); test('When there inbound transactions with amount less of Threshold, no alert should be created', async () => { @@ -501,7 +506,10 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(transactions[0]?.counterpartyBeneficiaryId); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual( + transactions[0]?.counterpartyBeneficiaryId, + ); }); test('When there are less than 5 inbound transactions with amount of 500, no alert should be created', async () => { @@ -608,7 +616,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual( + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual( business1Transactions[0]?.counterpartyOriginatorId, ); }); @@ -678,7 +687,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual( + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual( business1Transactions[0]?.counterpartyOriginatorId, ); }); @@ -742,7 +752,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual( + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual( business1Transactions[0]?.counterpartyOriginatorId, ); }); @@ -824,7 +835,8 @@ describe('AlertService', () => { }); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual( + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual( business1Transactions[0]?.counterpartyOriginatorId, ); }); @@ -904,7 +916,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual( + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual( chargebackTransactions[0]?.counterpartyOriginatorId, ); }); @@ -1014,7 +1027,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counteryparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counteryparty.id); }); it('When there are 2 credit card transactions with >100 base amount and one transaction exceeds the average of all credit card transactions, no alert should be created', async () => { @@ -1105,7 +1119,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counteryparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counteryparty.id); }); it('When there are 2 credit card transactions with >100 base amount and one transaction exceeds the average of all credit card transactions, no alert should be created', async () => { @@ -1333,7 +1348,7 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counteryparty.id); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counteryparty.id); }); it('When there are 2 credit card transactions with >100 base amount and one transaction exceeds the average of all credit card transactions, no alert should be created', async () => { @@ -1446,7 +1461,8 @@ describe('AlertService', () => { const alerts = await prismaService.alert.findMany(); expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counteryparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counteryparty.id); }); it('When there are 2 credit card transactions with >100 base amount and one transaction exceeds the average of all credit card transactions, no alert should be created', async () => { @@ -1506,7 +1522,7 @@ describe('AlertService', () => { // Has the customer been active for over 180 days (Has the customer had at least 1 Inbound credit card transactions more than 180 days ago)? (A condition that is used to ensure that we are calculating an average from an available representative sample of data - this condition would cause the rule not to alert in the customer's first 180 days of their credit card life cycle) // Has the customer had more than a set [Number] of Inbound credit card transactions within the last 3 days? (A condition that is used to exclude cases when the number of Inbound credit card transactions in 3 days is more than 2 times greater than the customer's 3-day historic average number of Inbound credit card transactions, although of an insignificantly low number) // Has the customer's number of Inbound credit card transactions in 3 days been more than a set [Factor] times greater than the customer's 3-day average number of Inbound credit card transactions (when the average is caclulated from the 177 days preceding the evaluated 3 days)? - it(`Trigger an alert when there inbound credit card transactions more than 180 days ago + it.only(`Trigger an alert when there inbound credit card transactions more than 180 days ago had more than a set X within the last 3 days`, async () => { // Arrange @@ -1518,11 +1534,8 @@ describe('AlertService', () => { ); oldDaysAgo.setHours(0, 0, 0, 0); - await oldTransactionFactory - .transactionDate(faker.date.recent(3, oldDaysAgo)) - .amount(3) - .count(1) - .create(); + const txDate = faker.date.recent(3, oldDaysAgo); + await oldTransactionFactory.transactionDate(txDate).amount(3).count(1).create(); // transactions from last days await oldTransactionFactory @@ -1564,9 +1577,7 @@ describe('AlertService', () => { hash: expect.any(String), }, executionRow: { - activedaystransactions: '60', - alltransactionscount: '186', - counterpartyId: counteryparty.id, + counterpartyBeneficiaryId: counteryparty.id, }, }); }); @@ -1683,7 +1694,7 @@ describe('AlertService', () => { executionRow: { activedaystransactions: '60', alltransactionscount: '186', - counterpartyId: counteryparty.id, + counterpartyBeneficiaryId: counteryparty.id, }, }); }); @@ -1767,14 +1778,15 @@ describe('AlertService', () => { expect(alerts[0]?.severity).toEqual('high'); - expect(alerts[0]?.counterpartyId).toEqual(counteryparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyOriginatorId).toEqual(counteryparty.id); expect(alerts[0]?.executionDetails).toMatchObject({ checkpoint: { hash: expect.any(String), }, executionRow: { - counterpartyId: counteryparty.id, + counterpartyOriginatorId: counteryparty.id, counterpertyInManyBusinessesCount: `${ ALERT_DEFINITIONS.MMOC_CC.inlineRule.options.minimumCount + 1 }`, @@ -2070,7 +2082,7 @@ describe('AlertService', () => { options: { ...ALERT_DEFINITIONS.DSTA_CC.inlineRule.options, timeAmount: 1, - timeUnit: 'days', + timeUnit: TIME_UNITS.days, direction: TransactionDirection.inbound, }, }, @@ -2108,7 +2120,8 @@ describe('AlertService', () => { expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counterparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counterparty.id); }); it(`Shouldnt create alert for non credit card transaction`, async () => { @@ -2209,7 +2222,7 @@ describe('AlertService', () => { options: { ...ALERT_DEFINITIONS.DSTA_APM.inlineRule.options, timeAmount: 1, - timeUnit: 'days', + timeUnit: TIME_UNITS.days, direction: TransactionDirection.inbound, }, }, @@ -2247,7 +2260,8 @@ describe('AlertService', () => { expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counterparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counterparty.id); }); it(`Shouldnt create alert for non credit card transaction`, async () => { @@ -2348,7 +2362,7 @@ describe('AlertService', () => { options: { ...ALERT_DEFINITIONS.DMT_CC.inlineRule.options, timeAmount: 1, - timeUnit: 'days', + timeUnit: TIME_UNITS.days, direction: TransactionDirection.inbound, }, }, @@ -2385,7 +2399,8 @@ describe('AlertService', () => { expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counterparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counterparty.id); }); it(`Shouldnt create alert for non credit card transaction`, async () => { @@ -2484,7 +2499,7 @@ describe('AlertService', () => { options: { ...ALERT_DEFINITIONS.DMT_APM.inlineRule.options, timeAmount: 1, - timeUnit: 'days', + timeUnit: TIME_UNITS.days, direction: TransactionDirection.inbound, }, }, @@ -2521,7 +2536,8 @@ describe('AlertService', () => { expect(alerts).toHaveLength(1); expect(alerts[0]?.alertDefinitionId).toEqual(alertDefinition.id); - expect(alerts[0]?.counterpartyId).toEqual(counterparty.id); + expect(alerts[0]?.counterpartyId).toEqual(null); + expect(alerts[0]?.counterpartyBeneficiaryId).toEqual(counterparty.id); }); it(`Shouldnt trigger alert for old transactions`, async () => { @@ -2551,6 +2567,7 @@ describe('AlertService', () => { }); }); }); + const createCounterparty = async ( prismaService: PrismaService, proj?: Pick, diff --git a/services/workflows-service/src/data-analytics/data-analytics.service.ts b/services/workflows-service/src/data-analytics/data-analytics.service.ts index 537cbaff0a..ac7e31467b 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.service.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.service.ts @@ -429,7 +429,7 @@ export class DataAnalyticsService { const query: Prisma.Sql = Prisma.sql` WITH transactions AS ( SELECT - "tr"."counterpartyBeneficiaryId", + "tr"."counterpartyBeneficiaryId" as "counterpartyBeneficiaryId", count( CASE WHEN "tr"."transactionDate" >= CURRENT_DATE - INTERVAL '${Prisma.raw( `${timeAmount} ${timeUnit}`, @@ -642,7 +642,7 @@ export class DataAnalyticsService { AND count(id) FILTER (WHERE "transactionDate" < ${historicalTransactionClause}) >= 1 ) SELECT - a."counterpartyBeneficiaryId" as "counterpartyId", + a."counterpartyBeneficiaryId" as "counterpartyBeneficiaryId", a.allTransactionsCount, a.activeDaysTransactions, a.lastTransactionsCount, @@ -689,7 +689,7 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti return await this._executeQuery>( Prisma.sql` SELECT - "tr"."counterpartyOriginatorId" as "counterpartyId", + "tr"."counterpartyOriginatorId" as "counterpartyOriginatorId", COUNT(distinct "tr"."counterpartyBeneficiaryId") as "counterpertyInManyBusinessesCount" FROM "TransactionRecord" as "tr" ${Prisma.join(uniqueJoinClause, '\n ')} @@ -736,7 +736,7 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti ].filter(Boolean); const sqlQuery = Prisma.sql`WITH tx_by_business AS - (SELECT "tr"."counterpartyBeneficiaryId" AS "counterpartyId", + (SELECT "tr"."counterpartyBeneficiaryId" as "counterpartyBeneficiaryId", "b"."businessType", COUNT("tr".id) FILTER ( WHERE ${transactionsOverAllTimeClause}) AS "transactionCount", @@ -755,13 +755,13 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti avg_business AS (SELECT "businessType", SUM("recentDaysTransactionCount") AS "totalTransactionsCount", - COUNT(DISTINCT "counterpartyId") AS "merchantCount" + COUNT(DISTINCT "counterpartyBeneficiaryId") AS "merchantCount" FROM tx_by_business WHERE "recentDaysTransactionCount" > ${minimumCount} GROUP BY "businessType" HAVING COUNT(*) > 1 AND SUM("recentDaysTransactionCount") > 1) - SELECT t."counterpartyId", + SELECT t."counterpartyBeneficiaryId", t."businessType", t."transactionCount", t."recentDaysTransactionCount", @@ -789,8 +789,6 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti excludePaymentMethods, transactionType = [], - - subjectColumn, }: DailySingleTransactionAmountType) { if (!projectId) { throw new Error('projectId is required'); @@ -830,18 +828,16 @@ AND a.activeDaysTransactions > ((a.lastTransactionsCount - a.activeDaysTransacti if (ruleType === 'amount') { conditions.push(Prisma.sql`"transactionBaseAmount" > ${amountThreshold}`); - query = Prisma.sql`SELECT ${Prisma.raw( - subjectColumn, - )} FROM "TransactionRecord" "tr" WHERE ${Prisma.join( + query = Prisma.sql`SELECT "counterpartyBeneficiaryId" FROM "TransactionRecord" "tr" WHERE ${Prisma.join( conditions, ' AND ', - )} GROUP BY "${Prisma.raw(subjectColumn)}"`; + )} GROUP BY "counterpartyBeneficiaryId"`; } else if (ruleType === 'count') { - query = Prisma.sql`SELECT ${Prisma.raw(subjectColumn)} + query = Prisma.sql`SELECT "counterpartyBeneficiaryId", COUNT(id) AS "transactionCount" FROM "TransactionRecord" "tr" WHERE ${Prisma.join( conditions, ' AND ', - )} GROUP BY ${Prisma.raw(subjectColumn)} HAVING ${Prisma.raw( + )} GROUP BY "counterpartyBeneficiaryId" HAVING ${Prisma.raw( `${AggregateType.COUNT}(id)`, )} > ${amountThreshold}`; } else { diff --git a/services/workflows-service/src/data-analytics/data-investigation.service.ts b/services/workflows-service/src/data-analytics/data-investigation.service.ts index 2e57a53cb6..904ffffd83 100644 --- a/services/workflows-service/src/data-analytics/data-investigation.service.ts +++ b/services/workflows-service/src/data-analytics/data-investigation.service.ts @@ -6,6 +6,7 @@ import { TIME_UNITS } from './consts'; import { DailySingleTransactionAmountType, HighTransactionTypePercentage, + HighVelocityHistoricAverageOptions, InlineRule, TCustomersTransactionTypeOptions, TDormantAccountOptions, @@ -71,15 +72,21 @@ export class DataInvestigationService { projectId, }); break; + case 'investigateHighVelocityHistoricAverage': + investigationFilter = this[inlineRule.fnInvestigationName]({ + ...inlineRule.options, + projectId, + }); + break; } if (!investigationFilter) { - this.logger.error(`No evaluation function found`, { + this.logger.error(`No investigation function found`, { inlineRule, }); throw new Error( - `No evaluation function found for rule name: ${(inlineRule as InlineRule).id}`, + `No investigation function found for rule name: ${(inlineRule as InlineRule).id}`, ); } @@ -251,6 +258,26 @@ export class DataInvestigationService { } as const satisfies Prisma.TransactionRecordWhereInput; } + async investigateHighVelocityHistoricAverage(options: HighVelocityHistoricAverageOptions) { + const { + projectId, + transactionDirection, + paymentMethod, + activeUserPeriod, + lastDaysPeriod, + timeUnit, + } = options; + + return { + projectId, + ...(transactionDirection ? { transactionDirection: transactionDirection } : {}), + paymentMethod: + paymentMethod.operator === '=' + ? { equals: paymentMethod.value } + : { not: paymentMethod.value }, + } as const satisfies Prisma.TransactionRecordWhereInput; + } + _buildTransactionsFiltersByAlert(inlineRule: InlineRule, alert?: Alert) { const whereClause: Prisma.TransactionRecordWhereInput = {}; diff --git a/services/workflows-service/src/data-analytics/types.ts b/services/workflows-service/src/data-analytics/types.ts index f084918178..f1dd25a204 100644 --- a/services/workflows-service/src/data-analytics/types.ts +++ b/services/workflows-service/src/data-analytics/types.ts @@ -10,7 +10,7 @@ export type InlineRule = { } & ( | { fnName: 'evaluateHighTransactionTypePercentage'; - fnInvestigationName?: 'investigateHighTransactionTypePercentage'; + fnInvestigationName: 'investigateHighTransactionTypePercentage'; options: Omit; } | { @@ -20,22 +20,22 @@ export type InlineRule = { } | { fnName: 'evaluateCustomersTransactionType'; - fnInvestigationName?: 'investigateCustomersTransactionType'; + fnInvestigationName: 'investigateCustomersTransactionType'; options: Omit; } | { fnName: 'evaluateTransactionAvg'; - fnInvestigationName?: 'investigateTransactionAvg'; + fnInvestigationName: 'investigateTransactionAvg'; options: Omit; } | { fnName: 'evaluateTransactionAvg'; - fnInvestigationName?: 'investigateTransactionAvg'; + fnInvestigationName: 'investigateTransactionAvg'; options: Omit; } | { fnName: 'evaluateDormantAccount'; - fnInvestigationName?: 'investigateDormantAccount'; + fnInvestigationName: 'investigateDormantAccount'; options: Omit; } | { @@ -45,22 +45,22 @@ export type InlineRule = { } | { fnName: 'evaluateHighVelocityHistoricAverage'; - fnInvestigationName?: 'investigateHighVelocityHistoricAverage'; + fnInvestigationName: 'investigateHighVelocityHistoricAverage'; options: Omit; } | { fnName: 'evaluateMultipleMerchantsOneCounterparty'; - fnInvestigationName?: 'investigateMultipleMerchantsOneCounterparty'; + fnInvestigationName: 'investigateMultipleMerchantsOneCounterparty'; options: Omit; } | { fnName: 'evaluateMerchantGroupAverage'; - fnInvestigationName?: 'investigateMerchantGroupAverage'; + fnInvestigationName: 'investigateMerchantGroupAverage'; options: Omit; } | { fnName: 'evaluateDailySingleTransactionAmount'; - fnInvestigationName?: 'investigateDailySingleTransactionAmount'; + fnInvestigationName: 'investigateDailySingleTransactionAmount'; options: Omit; } ); @@ -196,6 +196,4 @@ export type DailySingleTransactionAmountType = { paymentMethods: PaymentMethod[] | readonly PaymentMethod[]; excludePaymentMethods: boolean; - - subjectColumn: Subject; }; From 0f66025a844c442e6d502a9ac258c26f9c49b255 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 24 Nov 2024 00:05:45 +0200 Subject: [PATCH 09/18] chore: fix tests and all around --- .../workflows-service/src/alert/alert.service.intg.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/workflows-service/src/alert/alert.service.intg.test.ts b/services/workflows-service/src/alert/alert.service.intg.test.ts index 99216e97c2..bebe8162bd 100644 --- a/services/workflows-service/src/alert/alert.service.intg.test.ts +++ b/services/workflows-service/src/alert/alert.service.intg.test.ts @@ -1522,7 +1522,7 @@ describe('AlertService', () => { // Has the customer been active for over 180 days (Has the customer had at least 1 Inbound credit card transactions more than 180 days ago)? (A condition that is used to ensure that we are calculating an average from an available representative sample of data - this condition would cause the rule not to alert in the customer's first 180 days of their credit card life cycle) // Has the customer had more than a set [Number] of Inbound credit card transactions within the last 3 days? (A condition that is used to exclude cases when the number of Inbound credit card transactions in 3 days is more than 2 times greater than the customer's 3-day historic average number of Inbound credit card transactions, although of an insignificantly low number) // Has the customer's number of Inbound credit card transactions in 3 days been more than a set [Factor] times greater than the customer's 3-day average number of Inbound credit card transactions (when the average is caclulated from the 177 days preceding the evaluated 3 days)? - it.only(`Trigger an alert when there inbound credit card transactions more than 180 days ago + it(`Trigger an alert when there inbound credit card transactions more than 180 days ago had more than a set X within the last 3 days`, async () => { // Arrange @@ -1692,8 +1692,6 @@ describe('AlertService', () => { hash: expect.any(String), }, executionRow: { - activedaystransactions: '60', - alltransactionscount: '186', counterpartyBeneficiaryId: counteryparty.id, }, }); From 61e3eb39fcec102087d1c7b4f9785ff905da41ae Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 24 Nov 2024 00:08:33 +0200 Subject: [PATCH 10/18] chore: conflicts --- services/workflows-service/prisma/data-migrations | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/workflows-service/prisma/data-migrations b/services/workflows-service/prisma/data-migrations index f9008a19a5..38cbcd066f 160000 --- a/services/workflows-service/prisma/data-migrations +++ b/services/workflows-service/prisma/data-migrations @@ -1 +1 @@ -Subproject commit f9008a19a5430cead09300ac3e519d7189e8576c +Subproject commit 38cbcd066ff0af090600061a00999af959cd0e17 From 0c30b32956bd7f7a69e8591eba5601f0a893fd44 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Tue, 26 Nov 2024 23:40:30 +0200 Subject: [PATCH 11/18] chore: fix tests --- .../src/alert/alert.service.intg.test.ts | 1 - .../data-analytics/data-analytics.module.ts | 10 +-- .../test/helpers/create-alert-definition.ts | 52 +++------------- .../src/test/helpers/create-alert.ts | 49 +++------------ .../src/test/helpers/nest-app-helper.ts | 4 ++ ...ansaction.controller.external.intg.test.ts | 62 +++++++++---------- .../src/transaction/transaction.module.ts | 22 ++----- .../src/workflow/workflow.module.ts | 2 - 8 files changed, 60 insertions(+), 142 deletions(-) diff --git a/services/workflows-service/src/alert/alert.service.intg.test.ts b/services/workflows-service/src/alert/alert.service.intg.test.ts index bebe8162bd..f6400f4c9d 100644 --- a/services/workflows-service/src/alert/alert.service.intg.test.ts +++ b/services/workflows-service/src/alert/alert.service.intg.test.ts @@ -99,7 +99,6 @@ describe('AlertService', () => { AlertRepository, AlertDefinitionRepository, BusinessReportService, - BusinessReportService, AlertService, BusinessService, BusinessRepository, diff --git a/services/workflows-service/src/data-analytics/data-analytics.module.ts b/services/workflows-service/src/data-analytics/data-analytics.module.ts index 0f5f5c7b8c..543e892d98 100644 --- a/services/workflows-service/src/data-analytics/data-analytics.module.ts +++ b/services/workflows-service/src/data-analytics/data-analytics.module.ts @@ -6,20 +6,14 @@ import { DataAnalyticsControllerExternal } from '@/data-analytics/data-analytics import { PrismaModule } from '@/prisma/prisma.module'; import { ProjectScopeService } from '@/project/project-scope.service'; // eslint-disable-next-line import/no-cycle -import { BusinessReportModule } from '@/business-report/business-report.module'; // eslint-disable-next-line import/no-cycle import { AlertModule } from '@/alert/alert.module'; import { DataInvestigationService } from './data-investigation.service'; @Module({ - imports: [ - ACLModule, - PrismaModule, - forwardRef(() => BusinessReportModule), - forwardRef(() => AlertModule), - ], + imports: [ACLModule, PrismaModule, forwardRef(() => AlertModule)], controllers: [DataAnalyticsControllerInternal, DataAnalyticsControllerExternal], providers: [DataAnalyticsService, ProjectScopeService, DataInvestigationService], - exports: [ACLModule, DataAnalyticsService, DataInvestigationService], + exports: [DataAnalyticsService, DataInvestigationService], }) export class DataAnalyticsModule {} diff --git a/services/workflows-service/src/test/helpers/create-alert-definition.ts b/services/workflows-service/src/test/helpers/create-alert-definition.ts index 91be2a7880..7c525c993a 100644 --- a/services/workflows-service/src/test/helpers/create-alert-definition.ts +++ b/services/workflows-service/src/test/helpers/create-alert-definition.ts @@ -1,47 +1,18 @@ -import { Prisma } from '@prisma/client'; -import { faker } from '@faker-js/faker'; -import { AppLoggerService } from '../../common/app-logger/app-logger.service'; -import { PrismaService } from '../../prisma/prisma.service'; -import { Test } from '@nestjs/testing'; import { AlertService } from '@/alert/alert.service'; -import { AlertRepository } from '@/alert/alert.repository'; -import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; -import { AlertDefinitionRepository } from '@/alert-definition/alert-definition.repository'; -import { ProjectScopeService } from '@/project/project-scope.service'; -import { ClsService } from 'nestjs-cls'; +import { faker } from '@faker-js/faker'; +import { Prisma } from '@prisma/client'; import { merge } from 'lodash'; export const createAlertDefinition = async ( projectId: string, overrides: Prisma.AlertDefinitionCreateArgs = {} as Prisma.AlertDefinitionCreateArgs, + alertService: AlertService, ) => { - const moduleRef = await Test.createTestingModule({ - providers: [ - AlertService, - ClsService, - PrismaService, - DataAnalyticsService, - AlertDefinitionRepository, - ProjectScopeService, - AppLoggerService, - { - provide: AlertRepository, - useClass: AlertRepository, - }, - { - provide: 'LOGGER', - useValue: { - setContext: jest.fn(), - log: jest.fn(), - error: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - }, - }, - ], - }).compile(); - - const alertService = moduleRef.get(AlertService); + const fnName = faker.helpers.arrayElement([ + 'evaluateTransactionsAgainstDynamicRules', + 'evaluateDormantAccount', + 'evaluateMultipleMerchantsOneCounterparty', + ]); const definition = { crossEnvKey: faker.datatype.uuid(), name: faker.lorem.slug(), @@ -65,11 +36,8 @@ export const createAlertDefinition = async ( inlineRule: { id: faker.datatype.uuid(), - fnName: faker.helpers.arrayElement([ - 'evaluateTransactionsAgainstDynamicRules', - 'evaluateRiskScore', - 'evaluateCustomRule', - ]), + fnName, + fnInvestigationName: fnName.replace('evaluate', 'investigate'), options: { groupBy: [ faker.helpers.arrayElement([ diff --git a/services/workflows-service/src/test/helpers/create-alert.ts b/services/workflows-service/src/test/helpers/create-alert.ts index 1b56866ac5..15e50bdf6c 100644 --- a/services/workflows-service/src/test/helpers/create-alert.ts +++ b/services/workflows-service/src/test/helpers/create-alert.ts @@ -1,49 +1,16 @@ -import { AppLoggerService } from '../../common/app-logger/app-logger.service'; -import { PrismaService } from '../../prisma/prisma.service'; -import { AlertDefinition } from '@prisma/client'; -import { Test } from '@nestjs/testing'; import { AlertService } from '@/alert/alert.service'; -import { AlertRepository } from '@/alert/alert.repository'; -import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; -import { ClsService } from 'nestjs-cls'; -import { AlertDefinitionRepository } from '@/alert-definition/alert-definition.repository'; -import { ProjectScopeService } from '@/project/project-scope.service'; - -export const createAlert = async (projectId: string, AlertDefinition: AlertDefinition) => { - const moduleRef = await Test.createTestingModule({ - providers: [ - AlertService, - ClsService, - PrismaService, - DataAnalyticsService, - AlertDefinitionRepository, - ProjectScopeService, - AppLoggerService, - { - provide: AlertRepository, - useClass: AlertRepository, - }, - { - provide: 'LOGGER', - useValue: { - setContext: jest.fn(), - log: jest.fn(), - error: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - }, - }, - ], - }).compile(); - - const alertService = moduleRef.get(AlertService); +import { AlertDefinition } from '@prisma/client'; +export const createAlert = async ( + projectId: string, + alertDefinition: AlertDefinition, + alertService: AlertService, +) => { // Accessing private method for testing purposes while maintaining types return await alertService.createAlert( { - id: AlertDefinition.id, - projectId: AlertDefinition.projectId, - defaultSeverity: AlertDefinition.defaultSeverity, + ...alertDefinition, + projectId, }, [], {}, diff --git a/services/workflows-service/src/test/helpers/nest-app-helper.ts b/services/workflows-service/src/test/helpers/nest-app-helper.ts index d694d9fd88..c4b0a598fe 100644 --- a/services/workflows-service/src/test/helpers/nest-app-helper.ts +++ b/services/workflows-service/src/test/helpers/nest-app-helper.ts @@ -14,6 +14,8 @@ import { AuthKeyMiddleware } from '@/common/middlewares/auth-key.middleware'; import { CustomerModule } from '@/customer/customer.module'; import { HttpModule } from '@nestjs/axios'; import { ApiKeyService } from '@/customer/api-key/api-key.service'; +import { ConfigModule } from '@nestjs/config'; +import { EventEmitterModule } from '@nestjs/event-emitter'; export const commonTestingModules = [ ClsModule.forRoot({ @@ -22,6 +24,8 @@ export const commonTestingModules = [ AppLoggerModule, CustomerModule, HttpModule, + ConfigModule.forRoot({ isGlobal: true }), + EventEmitterModule.forRoot(), ]; export const fetchServiceFromModule = async ( diff --git a/services/workflows-service/src/transaction/transaction.controller.external.intg.test.ts b/services/workflows-service/src/transaction/transaction.controller.external.intg.test.ts index a3df994626..f1430945c7 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.intg.test.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.intg.test.ts @@ -37,6 +37,8 @@ import { DataAnalyticsService } from '@/data-analytics/data-analytics.service'; import { ConfigService } from '@nestjs/config'; import { AlertService } from '@/alert/alert.service'; import { MerchantMonitoringClient } from '@/business-report/merchant-monitoring-client'; +import { DataAnalyticsModule } from '@/data-analytics/data-analytics.module'; +import { AlertModule } from '@/alert/alert.module'; const getBusinessCounterpartyData = (business?: Business) => { if (business) { @@ -140,6 +142,7 @@ const API_KEY = faker.datatype.uuid(); describe('#TransactionControllerExternal', () => { let app: INestApplication; + let alertService: AlertService; let project: Project; let customer: Customer; @@ -148,20 +151,9 @@ describe('#TransactionControllerExternal', () => { beforeAll(async () => { await cleanupDatabase(); - app = await initiateNestApp( - app, - [ - ProjectScopeService, - AlertService, - AlertRepository, - AlertDefinitionRepository, - DataAnalyticsService, - ConfigService, - MerchantMonitoringClient, - ], - [TransactionControllerExternal], - [TransactionModule], - ); + app = await initiateNestApp(app, [], [], [TransactionModule, AlertModule]); + + alertService = app.get(AlertService); }); beforeEach(async () => { customer = await createCustomer( @@ -611,21 +603,23 @@ describe('#TransactionControllerExternal', () => { project = await createProject(app.get(PrismaService), customer, faker.datatype.uuid()); }); - const getAlertDefinitionWithTimeOptions = (timeUnit: string, timeAmount: number) => ({ - inlineRule: { - fnName: faker.helpers.arrayElement([ - 'evaluateMerchantGroupAverage', - 'evaluateHighTransactionTypePercentage', - 'evaluateTransactionsAgainstDynamicRules', - 'evaluateMultipleMerchantsOneCounterparty', - 'evaluateDormantAccount', - ]), - options: { - timeUnit, - timeAmount, + const getAlertDefinitionWithTimeOptions = (timeUnit: string, timeAmount: number) => { + const fnName = faker.helpers.arrayElement([ + 'evaluateMultipleMerchantsOneCounterparty', + 'evaluateDormantAccount', + ]); + + return { + inlineRule: { + fnName, + fnInvestigationName: fnName.replace('evaluate', 'investigate'), + options: { + timeUnit, + timeAmount, + }, }, - }, - }); + }; + }; const createTransactionWithDate = async (daysAgo: number) => { const currentDate = new Date(); @@ -639,8 +633,9 @@ describe('#TransactionControllerExternal', () => { alertDefinition = await createAlertDefinition( project.id, getAlertDefinitionWithTimeOptions('days', 7) as any, + alertService, ); - const alert = await createAlert(project.id, alertDefinition); + const alert = await createAlert(project.id, alertDefinition, alertService); await Promise.all([ // 5 transactions in the past 7 days @@ -677,8 +672,9 @@ describe('#TransactionControllerExternal', () => { alertDefinition = await createAlertDefinition( project.id, getAlertDefinitionWithTimeOptions('days', 1) as any, + alertService, ); - const alert = await createAlert(project.id, alertDefinition); + const alert = await createAlert(project.id, alertDefinition, alertService); // Create a transaction older than the alert criteria await createTransactionRecord(app.get(PrismaService), project, { @@ -711,9 +707,10 @@ describe('#TransactionControllerExternal', () => { alertDefinition = await createAlertDefinition( otherProject.id, getAlertDefinitionWithTimeOptions('days', 7) as any, + alertService, ); - const alert = await createAlert(otherProject.id, alertDefinition); + const alert = await createAlert(otherProject.id, alertDefinition, alertService); const response = await request(app.getHttpServer()) .get(`/external/transactions/by-alert?alertId=${alert.id}`) @@ -736,9 +733,10 @@ describe('#TransactionControllerExternal', () => { alertDefinition = await createAlertDefinition( project.id, getAlertDefinitionWithTimeOptions('days', 15) as any, + alertService, ); - const alert = await createAlert(project.id, alertDefinition); + const alert = await createAlert(project.id, alertDefinition, alertService); const response = await request(app.getHttpServer()) .get(`/external/transactions/by-alert?alertId=${alert.id}`) diff --git a/services/workflows-service/src/transaction/transaction.module.ts b/services/workflows-service/src/transaction/transaction.module.ts index afb8ff3996..a72f20ba3d 100644 --- a/services/workflows-service/src/transaction/transaction.module.ts +++ b/services/workflows-service/src/transaction/transaction.module.ts @@ -5,25 +5,15 @@ import { TransactionRepository } from '@/transaction/transaction.repository'; import { TransactionService } from '@/transaction/transaction.service'; import { TransactionControllerExternal } from '@/transaction/transaction.controller.external'; import { PrismaModule } from '@/prisma/prisma.module'; -import { ProjectScopeService } from '@/project/project-scope.service'; -import { SentryService } from '@/sentry/sentry.service'; -import { AlertService } from '@/alert/alert.service'; -import { AlertRepository } from '@/alert/alert.repository'; -import { AlertDefinitionRepository } from '@/alert-definition/alert-definition.repository'; import { DataAnalyticsModule } from '@/data-analytics/data-analytics.module'; +import { SentryModule } from '@/sentry/sentry.module'; +import { AlertModule } from '@/alert/alert.module'; +import { ProjectModule } from '@/project/project.module'; @Module({ - imports: [ACLModule, PrismaModule, DataAnalyticsModule], + imports: [ACLModule, PrismaModule, DataAnalyticsModule, SentryModule, AlertModule, ProjectModule], controllers: [TransactionControllerInternal, TransactionControllerExternal], - providers: [ - TransactionService, - TransactionRepository, - ProjectScopeService, - SentryService, - AlertService, - AlertRepository, - AlertDefinitionRepository, - ], - exports: [ACLModule, TransactionService], + providers: [TransactionService, TransactionRepository], + exports: [TransactionService], }) export class TransactionModule {} diff --git a/services/workflows-service/src/workflow/workflow.module.ts b/services/workflows-service/src/workflow/workflow.module.ts index 220ea14d5f..3e47acdd98 100644 --- a/services/workflows-service/src/workflow/workflow.module.ts +++ b/services/workflows-service/src/workflow/workflow.module.ts @@ -41,7 +41,6 @@ import { WorkflowService } from '@/workflow/workflow.service'; import { HttpModule } from '@nestjs/axios'; import { forwardRef, Module } from '@nestjs/common'; import { AlertModule } from '@/alert/alert.module'; -import { DataAnalyticsModule } from '@/data-analytics/data-analytics.module'; import { AlertDefinitionModule } from '@/alert-definition/alert-definition.module'; import { BusinessReportService } from '@/business-report/business-report.service'; import { RuleEngineModule } from '@/rule-engine/rule-engine.module'; @@ -61,7 +60,6 @@ import { SecretsManagerModule } from '@/secrets-manager/secrets-manager.module'; WorkflowDefinitionModule, AlertModule, BusinessModule, - DataAnalyticsModule, AlertDefinitionModule, RuleEngineModule, SecretsManagerModule, From 82fe11215106925fa8ad19d41a53a778e434ffed Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Tue, 26 Nov 2024 23:48:03 +0200 Subject: [PATCH 12/18] chore: changes --- .../workflows-service/src/transaction/transaction.module.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/workflows-service/src/transaction/transaction.module.ts b/services/workflows-service/src/transaction/transaction.module.ts index a72f20ba3d..0915da2f8c 100644 --- a/services/workflows-service/src/transaction/transaction.module.ts +++ b/services/workflows-service/src/transaction/transaction.module.ts @@ -14,6 +14,6 @@ import { ProjectModule } from '@/project/project.module'; imports: [ACLModule, PrismaModule, DataAnalyticsModule, SentryModule, AlertModule, ProjectModule], controllers: [TransactionControllerInternal, TransactionControllerExternal], providers: [TransactionService, TransactionRepository], - exports: [TransactionService], + exports: [ACLModule, TransactionService], }) export class TransactionModule {} From 493c25a3d79b3bf45f7bcbb1ea62b698f5b4156d Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 1 Dec 2024 13:55:45 +0200 Subject: [PATCH 13/18] fix: backward compatibility for alerts --- apps/backoffice-v2/src/common/utils/fetcher/fetcher.ts | 2 +- .../src/alert/alert.controller.external.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/backoffice-v2/src/common/utils/fetcher/fetcher.ts b/apps/backoffice-v2/src/common/utils/fetcher/fetcher.ts index 7c95e6f656..2e57879c56 100644 --- a/apps/backoffice-v2/src/common/utils/fetcher/fetcher.ts +++ b/apps/backoffice-v2/src/common/utils/fetcher/fetcher.ts @@ -69,7 +69,7 @@ export const fetcher: IFetcher = async ({ return await handlePromise(res.blob()); } - if (!res.headers.get('content-length') || Number(res.headers.get('content-length')) > 0) { + if (!res.headers.get('content-length') || Number(res.headers.get('content-length') || 0) > 0) { // TODO: make sure its json by checking the content-type in order to safe access to json method return await handlePromise(res.json()); } diff --git a/services/workflows-service/src/alert/alert.controller.external.ts b/services/workflows-service/src/alert/alert.controller.external.ts index 6be0efbe27..5e90968d8a 100644 --- a/services/workflows-service/src/alert/alert.controller.external.ts +++ b/services/workflows-service/src/alert/alert.controller.external.ts @@ -124,14 +124,17 @@ export class AlertControllerExternal { const { alertDefinition, assignee, + counterparty, counterpartyBeneficiary, counterpartyOriginator, state, ...alertWithoutDefinition } = alert as TAlertTransactionResponse; - const counterpartyDetails = (counterparty: TAlertTransactionResponse['counterparty']) => - counterparty.business + const counterpartyDetails = (counterparty: TAlertTransactionResponse['counterparty']) => { + if (!counterparty) return; + + return counterparty?.business ? { type: 'business', id: counterparty.business.id, @@ -144,7 +147,7 @@ export class AlertControllerExternal { correlationId: counterparty.endUser.correlationId, name: `${counterparty.endUser.firstName} ${counterparty.endUser.lastName}`, }; - + }; return { ...alertWithoutDefinition, correlationId: alertDefinition.correlationId, @@ -157,6 +160,7 @@ export class AlertControllerExternal { : null, alertDetails: alertDefinition.description, subject: + counterpartyDetails(counterparty) || counterpartyDetails(counterpartyBeneficiary) || counterpartyDetails(counterpartyOriginator), decision: state, From 29757736ddd33c945720c9eba0661a1565d33f79 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Fri, 6 Dec 2024 07:30:12 +0200 Subject: [PATCH 14/18] fix: backward compatibility for alerts --- .../src/alert/alert.controller.external.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/services/workflows-service/src/alert/alert.controller.external.ts b/services/workflows-service/src/alert/alert.controller.external.ts index 5e90968d8a..b882984445 100644 --- a/services/workflows-service/src/alert/alert.controller.external.ts +++ b/services/workflows-service/src/alert/alert.controller.external.ts @@ -76,6 +76,26 @@ export class AlertControllerExternal { avatarUrl: true, }, }, + counterparty: { + select: { + id: true, + business: { + select: { + id: true, + correlationId: true, + companyName: true, + }, + }, + endUser: { + select: { + id: true, + correlationId: true, + firstName: true, + lastName: true, + }, + }, + }, + }, counterpartyOriginator: { select: { id: true, From 15f43d9eccfcda018ac7c1356847522efa316538 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Fri, 6 Dec 2024 10:56:49 +0200 Subject: [PATCH 15/18] fix: backward compatibility for alerts --- services/workflows-service/scripts/alerts/generate-alerts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/workflows-service/scripts/alerts/generate-alerts.ts b/services/workflows-service/scripts/alerts/generate-alerts.ts index 0b98f39114..5fdbaa940f 100644 --- a/services/workflows-service/scripts/alerts/generate-alerts.ts +++ b/services/workflows-service/scripts/alerts/generate-alerts.ts @@ -255,7 +255,7 @@ export const ALERT_DEFINITIONS = { description: 'High Velocity - High number of inbound non-traditional payment transactions received from a Counterparty over a set period of time', inlineRule: { - id: 'HVIC_CC', + id: 'HVIC_APM', fnName: 'evaluateTransactionsAgainstDynamicRules', fnInvestigationName: 'investigateTransactionsAgainstDynamicRules', subjects: ['counterpartyBeneficiaryId', 'counterpartyOriginatorId'], From cef8ddf6270c5d65f5b6d2affa1429bd4de3ad6f Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 8 Dec 2024 01:20:46 +0200 Subject: [PATCH 16/18] fix: latest transcations fixes --- .../workflows-service/prisma/data-migrations | 2 +- .../src/alert/alert.controller.external.ts | 1 + .../src/alert/alert.service.intg.test.ts | 2 +- .../src/alert/alert.service.ts | 66 -------------- .../data-investigation.service.ts | 25 +++--- .../transaction.controller.external.ts | 85 ++++++++++--------- .../src/transaction/transaction.repository.ts | 33 ++++--- 7 files changed, 80 insertions(+), 134 deletions(-) diff --git a/services/workflows-service/prisma/data-migrations b/services/workflows-service/prisma/data-migrations index 3f28beab65..bfc772b0ad 160000 --- a/services/workflows-service/prisma/data-migrations +++ b/services/workflows-service/prisma/data-migrations @@ -1 +1 @@ -Subproject commit 3f28beab658a6e93feb30303b5f9f757489f6669 +Subproject commit bfc772b0ade3ae49465629d6c85ac26aac3796ab diff --git a/services/workflows-service/src/alert/alert.controller.external.ts b/services/workflows-service/src/alert/alert.controller.external.ts index b882984445..89f08f2b58 100644 --- a/services/workflows-service/src/alert/alert.controller.external.ts +++ b/services/workflows-service/src/alert/alert.controller.external.ts @@ -168,6 +168,7 @@ export class AlertControllerExternal { name: `${counterparty.endUser.firstName} ${counterparty.endUser.lastName}`, }; }; + return { ...alertWithoutDefinition, correlationId: alertDefinition.correlationId, diff --git a/services/workflows-service/src/alert/alert.service.intg.test.ts b/services/workflows-service/src/alert/alert.service.intg.test.ts index f6400f4c9d..ef3e116e4c 100644 --- a/services/workflows-service/src/alert/alert.service.intg.test.ts +++ b/services/workflows-service/src/alert/alert.service.intg.test.ts @@ -186,7 +186,7 @@ describe('AlertService', () => { ); const counterpartyBeneficiaryId = - baseTransactionFactory.data.counterpartyBeneficiary?.connect?.id; + baseTransactionFactory?.data?.counterpartyBeneficiary?.connect?.id; // Act await alertService.checkAllAlerts(); diff --git a/services/workflows-service/src/alert/alert.service.ts b/services/workflows-service/src/alert/alert.service.ts index 49ed9c1135..e4625a1a47 100644 --- a/services/workflows-service/src/alert/alert.service.ts +++ b/services/workflows-service/src/alert/alert.service.ts @@ -511,70 +511,4 @@ export class AlertService { return alertSeverityToNumber(a) < alertSeverityToNumber(b) ? 1 : -1; } - - buildTransactionsFiltersByAlert(alert: Alert & { alertDefinition: AlertDefinition }) { - const filters: { - endDate: Date | undefined; - startDate: Date | undefined; - } = { - endDate: undefined, - startDate: undefined, - }; - - const endDate = alert.dedupedAt || alert.createdAt; - endDate.setHours(23, 59, 59, 999); - filters.endDate = endDate; - - const inlineRule = alert?.alertDefinition?.inlineRule as InlineRule; - - // @ts-ignore - TODO: Replace logic with proper implementation for each rule - // eslint-disable-next-line - let { timeAmount, timeUnit } = inlineRule.options; - - if (!timeAmount || !timeUnit) { - if ( - inlineRule.fnName === 'evaluateHighVelocityHistoricAverage' && - inlineRule.options.lastDaysPeriod && - timeUnit - ) { - timeAmount = inlineRule.options.lastDaysPeriod.timeAmount; - } else { - return filters; - } - } - - let startDate = new Date(endDate); - - let subtractValue = 0; - - const baseSubstractByMin = timeAmount * 60 * 1000; - - switch (timeUnit) { - case TIME_UNITS.minutes: - subtractValue = baseSubstractByMin; - break; - case TIME_UNITS.hours: - subtractValue = 60 * baseSubstractByMin; - break; - case TIME_UNITS.days: - subtractValue = 24 * 60 * baseSubstractByMin; - break; - case TIME_UNITS.months: - startDate.setMonth(startDate.getMonth() - timeAmount); - break; - case TIME_UNITS.years: - startDate.setFullYear(startDate.getFullYear() - timeAmount); - break; - } - - startDate.setHours(0, 0, 0, 0); - startDate = new Date(startDate.getTime() - subtractValue); - - const oldestDate = new Date(Math.min(startDate.getTime(), new Date(alert.createdAt).getTime())); - - oldestDate.setHours(0, 0, 0, 0); - filters.startDate = oldestDate; - - return filters; - } } diff --git a/services/workflows-service/src/data-analytics/data-investigation.service.ts b/services/workflows-service/src/data-analytics/data-investigation.service.ts index 904ffffd83..36251a6755 100644 --- a/services/workflows-service/src/data-analytics/data-investigation.service.ts +++ b/services/workflows-service/src/data-analytics/data-investigation.service.ts @@ -78,16 +78,16 @@ export class DataInvestigationService { projectId, }); break; - } - - if (!investigationFilter) { - this.logger.error(`No investigation function found`, { - inlineRule, - }); + default: + this.logger.error(`No investigation filter obtained`, { + inlineRule, + }); - throw new Error( - `No investigation function found for rule name: ${(inlineRule as InlineRule).id}`, - ); + throw new Error( + `Investigation filter could not be obtained for rule id: ${ + (inlineRule as InlineRule).id + }`, + ); } return { @@ -290,7 +290,7 @@ export class DataInvestigationService { }; if (alert) { - const endDate = alert.updatedAt || alert.createdAt; + const endDate = alert.dedupedAt || alert.createdAt; endDate.setHours(23, 59, 59, 999); filters.endDate = endDate; } @@ -336,7 +336,10 @@ export class DataInvestigationService { } startDate.setHours(0, 0, 0, 0); - startDate = new Date(startDate.getTime() - subtractValue); + + if (subtractValue > 0) { + startDate = new Date(startDate.getTime() - subtractValue); + } if (filters.endDate) { startDate = new Date(Math.min(startDate.getTime(), filters.endDate.getTime())); diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index 19a7833159..e950c00d28 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -40,6 +40,8 @@ import { TransactionCreatedDto } from '@/transaction/dtos/transaction-created.dt import { PaymentMethod } from '@prisma/client'; import { isEmpty } from 'lodash'; import { TransactionEntityMapper } from './transaction.mapper'; +import { DataInvestigationService } from '@/data-analytics/data-investigation.service'; +import { InlineRule } from '@/data-analytics/types'; @swagger.ApiBearerAuth() @swagger.ApiTags('Transactions') @@ -52,6 +54,7 @@ export class TransactionControllerExternal { protected readonly logger: AppLoggerService, protected readonly alertService: AlertService, protected readonly dataAnalyticsService: DataAnalyticsService, + protected readonly dataInvestigationService: DataInvestigationService, ) {} @Post() @@ -348,20 +351,28 @@ export class TransactionControllerExternal { // Backward compatibility will be remove soon, if (isEmpty((alert.executionDetails as TExecutionDetails).filters)) { - return this.getTransactionsByAlertV1({ filters, projectId }); + return this.getTransactionsByAlertV1({ projectId, alert, filters }); } return this.getTransactionsByAlertV2({ projectId, alert, filters }); } private getTransactionsByAlertV1({ - filters, projectId, + alert, + filters, }: { - filters: GetTransactionsByAlertDto; projectId: string; + alert: NonNullable>>; + filters: Pick; }) { return this.service.getTransactionsV1(filters, projectId, { + // TODO: Better investigation for each rule + where: this.dataInvestigationService.getInvestigationFilter( + projectId, + alert.alertDefinition.inlineRule as InlineRule, + alert.executionDetails.subject, + ), include: { counterpartyBeneficiary: { select: { @@ -409,52 +420,50 @@ export class TransactionControllerExternal { filters, }: { projectId: string; - alert: Awaited>; + alert: NonNullable>>; filters: Pick; }) { - if (alert) { - return this.service.getTransactions(projectId, filters, { - where: alert.executionDetails.filters, - include: { - counterpartyBeneficiary: { - select: { - correlationId: true, - business: { - select: { - correlationId: true, - companyName: true, - }, + return this.service.getTransactions(projectId, filters, { + where: alert.executionDetails.filters, + include: { + counterpartyBeneficiary: { + select: { + correlationId: true, + business: { + select: { + correlationId: true, + companyName: true, }, - endUser: { - select: { - correlationId: true, - firstName: true, - lastName: true, - }, + }, + endUser: { + select: { + correlationId: true, + firstName: true, + lastName: true, }, }, }, - counterpartyOriginator: { - select: { - correlationId: true, - business: { - select: { - correlationId: true, - companyName: true, - }, + }, + counterpartyOriginator: { + select: { + correlationId: true, + business: { + select: { + correlationId: true, + companyName: true, }, - endUser: { - select: { - correlationId: true, - firstName: true, - lastName: true, - }, + }, + endUser: { + select: { + correlationId: true, + firstName: true, + lastName: true, }, }, }, }, - }); - } + }, + }); return []; } diff --git a/services/workflows-service/src/transaction/transaction.repository.ts b/services/workflows-service/src/transaction/transaction.repository.ts index 4ea0d0d72f..401533642d 100644 --- a/services/workflows-service/src/transaction/transaction.repository.ts +++ b/services/workflows-service/src/transaction/transaction.repository.ts @@ -103,9 +103,9 @@ export class TransactionRepository { projectId: string, options?: Prisma.TransactionRecordFindManyArgs, ): Promise { - const args = deepmerge(options || {}, { - where: this.buildFiltersV2(getTransactionsParameters), - }); + const _options = this.buildFindManyOptionsByFilter(getTransactionsParameters); + + const args = deepmerge(options || {}, _options); return this.prisma.transactionRecord.findMany( this.scopeService.scopeFindMany(args, [projectId]), @@ -147,22 +147,21 @@ export class TransactionRepository { } // eslint-disable-next-line ballerine/verify-repository-project-scoped - buildFiltersV2( - getTransactionsParameters: GetTransactionsDto, - ): Prisma.TransactionRecordWhereInput { - const args: Prisma.TransactionRecordFindManyArgs = { - ...TransactionRepository.buildTransactionPaginationArgs(getTransactionsParameters), - ...TransactionRepository.buildTransactionOrderByArgs(getTransactionsParameters), + buildFindManyOptionsByFilter(getTransactionsParameters: GetTransactionsDto) { + const transactionDate = { + ...(getTransactionsParameters.startDate && { gte: getTransactionsParameters.startDate }), + ...(getTransactionsParameters.endDate && { lte: getTransactionsParameters.endDate }), }; - const whereClause: Prisma.TransactionRecordWhereInput = { - transactionDate: { - gte: getTransactionsParameters.startDate, - lte: getTransactionsParameters.endDate, - }, - paymentMethod: getTransactionsParameters.paymentMethod, + return { + ...TransactionRepository.buildTransactionPaginationArgs(getTransactionsParameters), + ...TransactionRepository.buildTransactionOrderByArgs(getTransactionsParameters), + where: { + ...(Object.keys(transactionDate).length === 0 && transactionDate), + ...(getTransactionsParameters.paymentMethod && { + paymentMethod: getTransactionsParameters.paymentMethod, + }), + } as Prisma.TransactionRecordWhereInput satisfies Prisma.TransactionRecordWhereInput, }; - - return deepmerge(args, whereClause); } } From 361269737104e4c3d170fa09dde41689be9bb318 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 8 Dec 2024 01:37:32 +0200 Subject: [PATCH 17/18] fix: latest changes --- .../src/data-analytics/data-investigation.service.ts | 1 + .../src/transaction/transaction.controller.external.ts | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/services/workflows-service/src/data-analytics/data-investigation.service.ts b/services/workflows-service/src/data-analytics/data-investigation.service.ts index 36251a6755..7492854cab 100644 --- a/services/workflows-service/src/data-analytics/data-investigation.service.ts +++ b/services/workflows-service/src/data-analytics/data-investigation.service.ts @@ -355,6 +355,7 @@ export class DataInvestigationService { if (filters.endDate) { whereClause.transactionDate = { + ...(typeof whereClause.transactionDate === 'object' ? whereClause.transactionDate : {}), lte: filters.endDate, }; } diff --git a/services/workflows-service/src/transaction/transaction.controller.external.ts b/services/workflows-service/src/transaction/transaction.controller.external.ts index e950c00d28..ededfca4f7 100644 --- a/services/workflows-service/src/transaction/transaction.controller.external.ts +++ b/services/workflows-service/src/transaction/transaction.controller.external.ts @@ -464,7 +464,5 @@ export class TransactionControllerExternal { }, }, }); - - return []; } } From f2faf4b12b696d13e2c6763fb4dc6b1032786b38 Mon Sep 17 00:00:00 2001 From: Lior Zamir Date: Sun, 8 Dec 2024 01:54:06 +0200 Subject: [PATCH 18/18] fix: latest changes 2 --- .../src/data-analytics/data-investigation.service.ts | 2 +- .../workflows-service/src/transaction/transaction.repository.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/workflows-service/src/data-analytics/data-investigation.service.ts b/services/workflows-service/src/data-analytics/data-investigation.service.ts index 7492854cab..e150e85cec 100644 --- a/services/workflows-service/src/data-analytics/data-investigation.service.ts +++ b/services/workflows-service/src/data-analytics/data-investigation.service.ts @@ -258,7 +258,7 @@ export class DataInvestigationService { } as const satisfies Prisma.TransactionRecordWhereInput; } - async investigateHighVelocityHistoricAverage(options: HighVelocityHistoricAverageOptions) { + investigateHighVelocityHistoricAverage(options: HighVelocityHistoricAverageOptions) { const { projectId, transactionDirection, diff --git a/services/workflows-service/src/transaction/transaction.repository.ts b/services/workflows-service/src/transaction/transaction.repository.ts index 401533642d..30a074956d 100644 --- a/services/workflows-service/src/transaction/transaction.repository.ts +++ b/services/workflows-service/src/transaction/transaction.repository.ts @@ -157,7 +157,7 @@ export class TransactionRepository { ...TransactionRepository.buildTransactionPaginationArgs(getTransactionsParameters), ...TransactionRepository.buildTransactionOrderByArgs(getTransactionsParameters), where: { - ...(Object.keys(transactionDate).length === 0 && transactionDate), + ...(Object.keys(transactionDate).length > 0 && transactionDate), ...(getTransactionsParameters.paymentMethod && { paymentMethod: getTransactionsParameters.paymentMethod, }),