Skip to content

Commit

Permalink
move some logic from writeOneTrustAssessment to cli-sync-ot
Browse files Browse the repository at this point in the history
  • Loading branch information
@abrantesarthur
abrantesarthur committed Jan 14, 2025
1 parent 30f3a4c commit 239a72d
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 112 deletions.
162 changes: 103 additions & 59 deletions src/cli-sync-ot.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/usr/bin/env node
import { logger } from './logger';
import keyBy from 'lodash/keyBy';

import colors from 'colors';
import {
getListOfOneTrustAssessments,
Expand Down Expand Up @@ -28,75 +30,117 @@ import {
* yarn cli-sync-ot --hostname=customer.my.onetrust.com --auth=$ONE_TRUST_OAUTH_TOKEN --file=./oneTrustAssessment.json
*/
async function main(): Promise<void> {
const { file, fileFormat, hostname, auth, resource } =
const { file, fileFormat, hostname, auth, resource, debug } =
parseCliSyncOtArguments();

// try {
// TODO: move to helper function
if (resource === OneTrustPullResource.Assessments) {
// use the hostname and auth token to instantiate a client to talk to OneTrust
const oneTrust = createOneTrustGotInstance({ hostname, auth });

// fetch the list of all assessments in the OneTrust organization
const assessments = await getListOfOneTrustAssessments({ oneTrust });
try {
// TODO: move to helper function
if (resource === OneTrustPullResource.Assessments) {
// use the hostname and auth token to instantiate a client to talk to OneTrust
const oneTrust = createOneTrustGotInstance({ hostname, auth });

// fetch details about one assessment at a time and sync to disk right away to avoid running out of memory
await mapSeries(assessments, async (assessment, index) => {
logger.info(
`Fetching details about assessment ${index + 1} of ${
assessments.length
}...`,
);
const assessmentDetails = await getOneTrustAssessment({
oneTrust,
assessmentId: assessment.assessmentId,
});
// fetch the list of all assessments in the OneTrust organization
const assessments = await getListOfOneTrustAssessments({ oneTrust });

// enrich assessments with risk information
let riskDetails: OneTrustGetRiskResponse[] = [];
const riskIds = uniq(
assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) =>
s.questions.flatMap((q: OneTrustAssessmentQuestion) =>
(q.risks ?? []).flatMap((r) => r.riskId),
),
),
);
if (riskIds.length > 0) {
// fetch details about one assessment at a time and sync to disk right away to avoid running out of memory
await mapSeries(assessments, async (assessment, index) => {
logger.info(
`Fetching details about ${riskIds.length} risks for assessment ${
index + 1
} of ${assessments.length}...`,
`Fetching details about assessment ${index + 1} of ${
assessments.length
}...`,
);
riskDetails = await map(
riskIds,
(riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }),
{
concurrency: 5,
},
const assessmentDetails = await getOneTrustAssessment({
oneTrust,
assessmentId: assessment.assessmentId,
});

// enrich assessments with risk information
let riskDetails: OneTrustGetRiskResponse[] = [];
const riskIds = uniq(
assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) =>
s.questions.flatMap((q: OneTrustAssessmentQuestion) =>
(q.risks ?? []).flatMap((r) => r.riskId),
),
),
);
}
if (riskIds.length > 0) {
logger.info(
`Fetching details about ${riskIds.length} risks for assessment ${
index + 1
} of ${assessments.length}...`,
);
riskDetails = await map(
riskIds,
(riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }),
{
concurrency: 5,
},
);
}

// enrich the sections with risk details
const riskDetailsById = keyBy(riskDetails, 'id');
const { sections, ...restAssessmentDetails } = assessmentDetails;
const enrichedSections = sections.map((section) => {
const { questions, ...restSection } = section;
const enrichedQuestions = questions.map((question) => {
const { risks, ...restQuestion } = question;
const enrichedRisks = (risks ?? []).map((risk) => {
const details = riskDetailsById[risk.riskId];
// TODO: missing the risk meta data and links to the assessment
return {
...risk,
description: details.description,
name: details.name,
treatment: details.treatment,
treatmentStatus: details.treatmentStatus,
type: details.type,
state: details.state,
stage: details.stage,
result: details.result,
categories: details.categories,
};
});
return {
...restQuestion,
risks: enrichedRisks,
};
});
return {
...restSection,
questions: enrichedQuestions,
};
});

writeOneTrustAssessment({
assessment,
assessmentDetails,
riskDetails,
index,
total: assessments.length,
file,
fileFormat,
// combine the two assessments into a single enriched result
const enrichedAssessment = {
...restAssessmentDetails,
sections: enrichedSections,
};

writeOneTrustAssessment({
assessment: {
...assessment,
...enrichedAssessment,
},
riskDetails,
index,
total: assessments.length,
file,
fileFormat,
});
});
});
}
} catch (err) {
logger.error(
colors.red(
`An error occurred pulling the resource ${resource} from OneTrust: ${
debug ? err.stack : err.message
}`,
),
);
process.exit(1);
}
// } catch (err) {
// logger.error(
// colors.red(
// `An error occurred pulling the resource ${resource} from OneTrust: ${
// debug ? err.stack : err.message
// }`,
// ),
// );
// process.exit(1);
// }

// Indicate success
logger.info(
Expand Down
57 changes: 4 additions & 53 deletions src/oneTrust/writeOneTrustAssessment.ts
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
import { logger } from '../logger';
import keyBy from 'lodash/keyBy';
import colors from 'colors';
import { OneTrustFileFormat } from '../enums';
import fs from 'fs';
import { flattenOneTrustAssessment } from './flattenOneTrustAssessment';
import { DEFAULT_ONE_TRUST_ASSESSMENT_CSV_HEADER } from './constants';
import { decodeCodec } from '@transcend-io/type-utils';
import {
OneTrustAssessment,
OneTrustAssessmentCsvRecord,
OneTrustGetAssessmentResponse,
OneTrustGetRiskResponse,
} from '@transcend-io/privacy-types';
import { OneTrustCombinedAssessment } from './codecs';

/**
* Write the assessment to disk at the specified file path.
Expand All @@ -23,8 +21,6 @@ export const writeOneTrustAssessment = ({
file,
fileFormat,
assessment,
assessmentDetails,
riskDetails,
index,
total,
}: {
Expand All @@ -33,9 +29,7 @@ export const writeOneTrustAssessment = ({
/** The format of the output file */
fileFormat: OneTrustFileFormat;
/** The basic assessment */
assessment: OneTrustAssessment;
/** The assessment with details */
assessmentDetails: OneTrustGetAssessmentResponse;
assessment: OneTrustCombinedAssessment;
/** The details of risks found within the assessment */
riskDetails: OneTrustGetRiskResponse[];
/** The index of the assessment being written to the file */
Expand All @@ -51,54 +45,14 @@ export const writeOneTrustAssessment = ({
),
);

// enrich the sections with risk details
const riskDetailsById = keyBy(riskDetails, 'id');
const { sections, ...restAssessmentDetails } = assessmentDetails;
const enrichedSections = sections.map((section) => {
const { questions, ...restSection } = section;
const enrichedQuestions = questions.map((question) => {
const { risks, ...restQuestion } = question;
const enrichedRisks = (risks ?? []).map((risk) => {
const details = riskDetailsById[risk.riskId];
// TODO: missing the risk meta data and links to the assessment
return {
...risk,
description: details.description,
name: details.name,
treatment: details.treatment,
treatmentStatus: details.treatmentStatus,
type: details.type,
state: details.state,
stage: details.stage,
result: details.result,
categories: details.categories,
};
});
return {
...restQuestion,
risks: enrichedRisks,
};
});
return {
...restSection,
questions: enrichedQuestions,
};
});

// combine the two assessments into a single enriched result
const enrichedAssessment = {
...restAssessmentDetails,
sections: enrichedSections,
};

// For json format
if (fileFormat === OneTrustFileFormat.Json) {
// start with an opening bracket
if (index === 0) {
fs.writeFileSync(file, '[\n');
}

const stringifiedAssessment = JSON.stringify(enrichedAssessment, null, 2);
const stringifiedAssessment = JSON.stringify(assessment, null, 2);

// Add comma for all items except the last one
const comma = index < total - 1 ? ',' : '';
Expand All @@ -119,10 +73,7 @@ export const writeOneTrustAssessment = ({
}

// flatten the assessment object so it does not have nested properties
const flatAssessment = flattenOneTrustAssessment({
...assessment,
...enrichedAssessment,
});
const flatAssessment = flattenOneTrustAssessment(assessment);

// comment
const flatAssessmentFull = Object.fromEntries(
Expand Down

0 comments on commit 239a72d

Please sign in to comment.