From 239a72d535072cb375d813bfcd4f4aabda27c62b Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Jan 2025 02:20:27 +0000 Subject: [PATCH] move some logic from writeOneTrustAssessment to cli-sync-ot --- src/cli-sync-ot.ts | 162 +++++++++++++++--------- src/oneTrust/writeOneTrustAssessment.ts | 57 +-------- 2 files changed, 107 insertions(+), 112 deletions(-) diff --git a/src/cli-sync-ot.ts b/src/cli-sync-ot.ts index 771418c3..da163e82 100644 --- a/src/cli-sync-ot.ts +++ b/src/cli-sync-ot.ts @@ -1,5 +1,7 @@ #!/usr/bin/env node import { logger } from './logger'; +import keyBy from 'lodash/keyBy'; + import colors from 'colors'; import { getListOfOneTrustAssessments, @@ -28,75 +30,117 @@ import { * yarn cli-sync-ot --hostname=customer.my.onetrust.com --auth=$ONE_TRUST_OAUTH_TOKEN --file=./oneTrustAssessment.json */ async function main(): Promise { - const { file, fileFormat, hostname, auth, resource } = + const { file, fileFormat, hostname, auth, resource, debug } = parseCliSyncOtArguments(); - // try { - // TODO: move to helper function - if (resource === OneTrustPullResource.Assessments) { - // use the hostname and auth token to instantiate a client to talk to OneTrust - const oneTrust = createOneTrustGotInstance({ hostname, auth }); - - // fetch the list of all assessments in the OneTrust organization - const assessments = await getListOfOneTrustAssessments({ oneTrust }); + try { + // TODO: move to helper function + if (resource === OneTrustPullResource.Assessments) { + // use the hostname and auth token to instantiate a client to talk to OneTrust + const oneTrust = createOneTrustGotInstance({ hostname, auth }); - // fetch details about one assessment at a time and sync to disk right away to avoid running out of memory - await mapSeries(assessments, async (assessment, index) => { - logger.info( - `Fetching details about assessment ${index + 1} of ${ - assessments.length - }...`, - ); - const assessmentDetails = await getOneTrustAssessment({ - oneTrust, - assessmentId: assessment.assessmentId, - }); + // fetch the list of all assessments in the OneTrust organization + const assessments = await getListOfOneTrustAssessments({ oneTrust }); - // enrich assessments with risk information - let riskDetails: OneTrustGetRiskResponse[] = []; - const riskIds = uniq( - assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) => - s.questions.flatMap((q: OneTrustAssessmentQuestion) => - (q.risks ?? []).flatMap((r) => r.riskId), - ), - ), - ); - if (riskIds.length > 0) { + // fetch details about one assessment at a time and sync to disk right away to avoid running out of memory + await mapSeries(assessments, async (assessment, index) => { logger.info( - `Fetching details about ${riskIds.length} risks for assessment ${ - index + 1 - } of ${assessments.length}...`, + `Fetching details about assessment ${index + 1} of ${ + assessments.length + }...`, ); - riskDetails = await map( - riskIds, - (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }), - { - concurrency: 5, - }, + const assessmentDetails = await getOneTrustAssessment({ + oneTrust, + assessmentId: assessment.assessmentId, + }); + + // enrich assessments with risk information + let riskDetails: OneTrustGetRiskResponse[] = []; + const riskIds = uniq( + assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) => + s.questions.flatMap((q: OneTrustAssessmentQuestion) => + (q.risks ?? []).flatMap((r) => r.riskId), + ), + ), ); - } + if (riskIds.length > 0) { + logger.info( + `Fetching details about ${riskIds.length} risks for assessment ${ + index + 1 + } of ${assessments.length}...`, + ); + riskDetails = await map( + riskIds, + (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }), + { + concurrency: 5, + }, + ); + } + + // enrich the sections with risk details + const riskDetailsById = keyBy(riskDetails, 'id'); + const { sections, ...restAssessmentDetails } = assessmentDetails; + const enrichedSections = sections.map((section) => { + const { questions, ...restSection } = section; + const enrichedQuestions = questions.map((question) => { + const { risks, ...restQuestion } = question; + const enrichedRisks = (risks ?? []).map((risk) => { + const details = riskDetailsById[risk.riskId]; + // TODO: missing the risk meta data and links to the assessment + return { + ...risk, + description: details.description, + name: details.name, + treatment: details.treatment, + treatmentStatus: details.treatmentStatus, + type: details.type, + state: details.state, + stage: details.stage, + result: details.result, + categories: details.categories, + }; + }); + return { + ...restQuestion, + risks: enrichedRisks, + }; + }); + return { + ...restSection, + questions: enrichedQuestions, + }; + }); - writeOneTrustAssessment({ - assessment, - assessmentDetails, - riskDetails, - index, - total: assessments.length, - file, - fileFormat, + // combine the two assessments into a single enriched result + const enrichedAssessment = { + ...restAssessmentDetails, + sections: enrichedSections, + }; + + writeOneTrustAssessment({ + assessment: { + ...assessment, + ...enrichedAssessment, + }, + riskDetails, + index, + total: assessments.length, + file, + fileFormat, + }); }); - }); + } + } catch (err) { + logger.error( + colors.red( + `An error occurred pulling the resource ${resource} from OneTrust: ${ + debug ? err.stack : err.message + }`, + ), + ); + process.exit(1); } - // } catch (err) { - // logger.error( - // colors.red( - // `An error occurred pulling the resource ${resource} from OneTrust: ${ - // debug ? err.stack : err.message - // }`, - // ), - // ); - // process.exit(1); - // } // Indicate success logger.info( diff --git a/src/oneTrust/writeOneTrustAssessment.ts b/src/oneTrust/writeOneTrustAssessment.ts index b674eade..eb7a08fc 100644 --- a/src/oneTrust/writeOneTrustAssessment.ts +++ b/src/oneTrust/writeOneTrustAssessment.ts @@ -1,5 +1,4 @@ import { logger } from '../logger'; -import keyBy from 'lodash/keyBy'; import colors from 'colors'; import { OneTrustFileFormat } from '../enums'; import fs from 'fs'; @@ -7,11 +6,10 @@ import { flattenOneTrustAssessment } from './flattenOneTrustAssessment'; import { DEFAULT_ONE_TRUST_ASSESSMENT_CSV_HEADER } from './constants'; import { decodeCodec } from '@transcend-io/type-utils'; import { - OneTrustAssessment, OneTrustAssessmentCsvRecord, - OneTrustGetAssessmentResponse, OneTrustGetRiskResponse, } from '@transcend-io/privacy-types'; +import { OneTrustCombinedAssessment } from './codecs'; /** * Write the assessment to disk at the specified file path. @@ -23,8 +21,6 @@ export const writeOneTrustAssessment = ({ file, fileFormat, assessment, - assessmentDetails, - riskDetails, index, total, }: { @@ -33,9 +29,7 @@ export const writeOneTrustAssessment = ({ /** The format of the output file */ fileFormat: OneTrustFileFormat; /** The basic assessment */ - assessment: OneTrustAssessment; - /** The assessment with details */ - assessmentDetails: OneTrustGetAssessmentResponse; + assessment: OneTrustCombinedAssessment; /** The details of risks found within the assessment */ riskDetails: OneTrustGetRiskResponse[]; /** The index of the assessment being written to the file */ @@ -51,46 +45,6 @@ export const writeOneTrustAssessment = ({ ), ); - // enrich the sections with risk details - const riskDetailsById = keyBy(riskDetails, 'id'); - const { sections, ...restAssessmentDetails } = assessmentDetails; - const enrichedSections = sections.map((section) => { - const { questions, ...restSection } = section; - const enrichedQuestions = questions.map((question) => { - const { risks, ...restQuestion } = question; - const enrichedRisks = (risks ?? []).map((risk) => { - const details = riskDetailsById[risk.riskId]; - // TODO: missing the risk meta data and links to the assessment - return { - ...risk, - description: details.description, - name: details.name, - treatment: details.treatment, - treatmentStatus: details.treatmentStatus, - type: details.type, - state: details.state, - stage: details.stage, - result: details.result, - categories: details.categories, - }; - }); - return { - ...restQuestion, - risks: enrichedRisks, - }; - }); - return { - ...restSection, - questions: enrichedQuestions, - }; - }); - - // combine the two assessments into a single enriched result - const enrichedAssessment = { - ...restAssessmentDetails, - sections: enrichedSections, - }; - // For json format if (fileFormat === OneTrustFileFormat.Json) { // start with an opening bracket @@ -98,7 +52,7 @@ export const writeOneTrustAssessment = ({ fs.writeFileSync(file, '[\n'); } - const stringifiedAssessment = JSON.stringify(enrichedAssessment, null, 2); + const stringifiedAssessment = JSON.stringify(assessment, null, 2); // Add comma for all items except the last one const comma = index < total - 1 ? ',' : ''; @@ -119,10 +73,7 @@ export const writeOneTrustAssessment = ({ } // flatten the assessment object so it does not have nested properties - const flatAssessment = flattenOneTrustAssessment({ - ...assessment, - ...enrichedAssessment, - }); + const flatAssessment = flattenOneTrustAssessment(assessment); // comment const flatAssessmentFull = Object.fromEntries(