Restructure CCI to NIST and NIST to CCI logic across mappers and dele…

…te obsolete files

Signed-off-by: Joyce Quach <[email protected]>

.github/workflows/convert-cci-list.yml

@@ -48,7 +48,7 @@ jobs:
           echo "OUTPUT_DIRECTORY=$ROOT_DIRECTORY/libs/hdf-converters/src/mappings" >> $GITHUB_ENV
 
       - name: Convert CCI List XML to CCI->NIST, CCI->Definitions, and NIST->CCI JSON files
-        run: yarn workspace @mitre/hdf-converters cciListXml2json $ROOT_DIRECTORY/U_CCI_List.xml $OUTPUT_DIRECTORY/U_CCI_List.nist.json $OUTPUT_DIRECTORY/U_CCI_List.defs.json $OUTPUT_DIRECTORY/U_CCI_List.cci.json
+        run: yarn workspace @mitre/hdf-converters cciListXml2json -i $ROOT_DIRECTORY/U_CCI_List.xml -n $OUTPUT_DIRECTORY/U_CCI_List.nist.json -d $OUTPUT_DIRECTORY/U_CCI_List.defs.json -c $OUTPUT_DIRECTORY/U_CCI_List.cci.json
 
       - name: Commit changes to CciNistMappingData.ts
         # run: |

libs/hdf-converters/data/converters/cciListXml2json.ts

@@ -1,13 +1,29 @@
 import fs from 'fs';
 import * as _ from 'lodash';
 import xml2js from 'xml2js';
+import {parseArgs} from 'node:util';
 
 // Documentation is located at https://github.com/mitre/heimdall2/wiki/Control-Correlation-Identifier-(CCI)-Converter.
 const parser = new xml2js.Parser();
-const pathToInfile = process.argv[2];
-const pathToCci2NistOutfile = process.argv[3];
-const pathToCci2DefinitionsOutfile = process.argv[4];
-const pathToNist2CciOutfile = process.argv[5];
+
+const options = {
+  input: {
+    type: 'string',
+    short: 'i'
+  },
+  cci2nist: {
+    type: 'string',
+    short: 'n'
+  },
+  cci2definitions: {
+    type: 'string',
+    short: 'd'
+  },
+  nist2cci: {
+    type: 'string',
+    short: 'c'
+  }
+} as const;
 
 // XML Structure after conversion
 export interface ICCIList {
@@ -26,61 +42,106 @@ export interface ICCIList {
   };
 }
 
-if (
-  !pathToInfile ||
-  !pathToCci2NistOutfile ||
-  !pathToCci2DefinitionsOutfile ||
-  !pathToNist2CciOutfile
-) {
-  console.error(`You must provide the path to the input and two output files.`);
-} else {
-  fs.readFile(pathToInfile, function (readFileError, data) {
-    if (readFileError) {
-      console.error(`Failed to read ${pathToInfile}: ${readFileError}`);
-    } else {
-      // Parse XML to JS Object
-      parser.parseString(data, (parseFileError: any, converted: ICCIList) => {
-        if (parseFileError) {
-          console.error(`Failed to parse ${pathToInfile}: ${parseFileError}`);
-        } else {
-          // These store our CCI->NIST names, CCI->definitions, and NIST->CCI mappings
-          const nists: Record<string, string> = {};
-          const definitions: Record<string, string> = {};
-          const ccis: Record<string, string[]> = {};
+// Check that we're not doing `npm test`; it will look for the arguments to the input and output files.
+const scriptIsCalled = process.argv[1].includes('cciListXml2json');
 
-          // For all CCI items
-          for (const cciItem of converted.cci_list.cci_items[0].cci_item) {
-            // Get the latest reference
-            const newestReference = _.maxBy(
-              cciItem.references?.[0].reference,
-              (item) => _.get(item, '$.version')
-            );
-            if (newestReference) {
-              nists[cciItem.$.id] = newestReference.$.index;
-              if (ccis[newestReference.$.index] === undefined) {
-                ccis[newestReference.$.index] = [cciItem.$.id];
+if (scriptIsCalled) {
+  const {values} = parseArgs({options});
+
+  const pathToInfile = values.input;
+  const pathToCci2NistOutfile = values.cci2nist;
+  const pathToCci2DefinitionsOutfile = values.cci2definitions;
+  const pathToNist2CciOutfile = values.nist2cci;
+
+  if (
+    !pathToInfile ||
+    !pathToCci2NistOutfile ||
+    !pathToCci2DefinitionsOutfile ||
+    !pathToNist2CciOutfile
+  ) {
+    console.error(
+      'You must provide the path to the input and three output files.'
+    );
+  } else {
+    fs.readFile(pathToInfile, function (readFileError, data) {
+      if (readFileError) {
+        console.error(`Failed to read ${pathToInfile}: ${readFileError}`);
+      } else {
+        // Parse XML to JS Object
+        parser.parseString(data, (parseFileError: any, converted: ICCIList) => {
+          if (parseFileError) {
+            console.error(`Failed to parse ${pathToInfile}: ${parseFileError}`);
+          } else {
+            // These store our CCI->NIST names, CCI->definitions, and NIST->CCI mappings
+            const nists: Record<string, string> = {};
+            const definitions: Record<string, string> = {};
+            const ccis: Record<string, string[]> = {};
+
+            // For all CCI items
+            for (const cciItem of converted.cci_list.cci_items[0].cci_item) {
+              // Get the latest reference
+              const newestReference = _.maxBy(
+                cciItem.references?.[0].reference,
+                (item) => _.get(item, '$.version')
+              );
+              if (newestReference) {
+                nists[cciItem.$.id] = newestReference.$.index;
+                if (ccis[newestReference.$.index] === undefined) {
+                  ccis[newestReference.$.index] = [cciItem.$.id];
+                } else {
+                  ccis[newestReference.$.index].push(cciItem.$.id);
+                }
+                definitions[cciItem.$.id] = cciItem.definition[0];
               } else {
-                ccis[newestReference.$.index].push(cciItem.$.id);
+                console.error(`No NIST Controls found for ${cciItem.$.id}`);
               }
-              definitions[cciItem.$.id] = cciItem.definition[0];
-            } else {
-              console.error(`No NIST Controls found for ${cciItem.$.id}`);
             }
+            fs.writeFileSync(
+              pathToCci2NistOutfile,
+              JSON.stringify(nists, null, 2)
+            );
+            fs.writeFileSync(
+              pathToCci2DefinitionsOutfile,
+              JSON.stringify(definitions, null, 2)
+            );
+            fs.writeFileSync(
+              pathToNist2CciOutfile,
+              JSON.stringify(unflatten(ccis), null, 2)
+            );
           }
-          fs.writeFileSync(
-            pathToCci2NistOutfile,
-            JSON.stringify(nists, null, 2)
-          );
-          fs.writeFileSync(
-            pathToCci2DefinitionsOutfile,
-            JSON.stringify(definitions, null, 2)
-          );
-          fs.writeFileSync(
-            pathToNist2CciOutfile,
-            JSON.stringify(ccis, null, 2)
-          );
-        }
-      });
-    }
-  });
+        });
+      }
+    });
+  }
+}
+
+const CCIS_KEY = 'ccis';
+
+type Leaf = {
+  [CCIS_KEY]?: string[];
+};
+
+type Branch = Leaf & {
+  [key: string]: Branch | string[] | undefined;
+};
+
+export type Trie = {
+  [key: string]: Branch;
+};
+
+export function removeParentheses(key: string): string {
+  return key.replace(/[()]/g, '');
+}
+
+function unflatten(fullNistPathToListOfCcis: Record<string, string[]>): Trie {
+  const DELIMITER = ' ';
+  const result = {};
+  const keys = _.keys(fullNistPathToListOfCcis);
+  for (const key of keys) {
+    const path = key.split(DELIMITER).map(removeParentheses);
+    path.push(CCIS_KEY);
+    const value = fullNistPathToListOfCcis[key];
+    _.setWith(result, path, value, Object);
+  }
+  return result;
 }

libs/hdf-converters/src/asff-mapper/asff-mapper.ts

@@ -7,7 +7,6 @@ import {ExecJSON} from 'inspecjs';
 import * as _ from 'lodash';
 import {version as HeimdallToolsVersion} from '../../package.json';
 import {BaseConverter, ILookupPath, MappedTransform} from '../base-converter';
-import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from '../utils/global';
 import {getCMSInSpec} from './case-cms-inspec';
 import {getFirewallManager} from './case-firewall-manager';
 import {getGuardDuty} from './case-guardduty';
@@ -16,7 +15,8 @@ import {getPreviouslyHDF} from './case-previously-hdf';
 import {getProwler} from './case-prowler';
 import {getSecurityHub} from './case-security-hub';
 import {getTrivy} from './case-trivy';
-import {getCCIsForNISTTags} from '../mappings/CciNistMapping';
+import {NIST2CCI} from '../mappings/CciNistMapping';
+import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from '../mappings/CciNistMappingData';
 
 const IMPACT_MAPPING: Map<string, number> = new Map([
   ['CRITICAL', 0.9],
@@ -455,11 +455,9 @@ export class ASFFMapper extends BaseConverter {
                     []
                   ) as string[];
                   if (tags.length === 0) {
-                    return getCCIsForNISTTags(
-                      DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS
-                    );
+                    return NIST2CCI(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS);
                   } else {
-                    return getCCIsForNISTTags(tags);
+                    return NIST2CCI(tags);
                   }
                 }
               },

libs/hdf-converters/src/asff-mapper/case-previously-hdf.ts

@@ -4,10 +4,10 @@ import * as _ from 'lodash';
 import {ILookupPath, MappedTransform} from '../base-converter';
 import {
   conditionallyProvideAttribute,
-  DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS,
   FROM_ASFF_TYPES_SLASH_REPLACEMENT
 } from '../utils/global';
 import {ASFFMapper, consolidate, SpecialCasing} from './asff-mapper';
+import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from '../mappings/CciNistMappingData';
 
 function replaceTypesSlashes<T>(type: T): T | string {
   if (!_.isString(type)) {

libs/hdf-converters/src/asff-mapper/case-trivy.ts

@@ -1,7 +1,7 @@
 import {encode} from 'html-entities';
 import {ExecJSON} from 'inspecjs';
 import * as _ from 'lodash';
-import {DEFAULT_UPDATE_REMEDIATION_NIST_TAGS} from '../utils/global';
+import {DEFAULT_UPDATE_REMEDIATION_NIST_TAGS} from '../mappings/CciNistMappingData';
 
 function findingId(finding: unknown): string {
   const generatorId = _.get(finding, 'GeneratorId');

libs/hdf-converters/src/burpsuite-mapper.ts

@@ -10,8 +10,8 @@ import {
   parseXml
 } from './base-converter';
 import {CweNistMapping} from './mappings/CweNistMapping';
-import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from './utils/global';
-import {getCCIsForNISTTags} from './mappings/CciNistMapping';
+import {NIST2CCI} from './mappings/CciNistMapping';
+import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from './mappings/CciNistMappingData';
 
 // Constant
 const IMPACT_MAPPING: Map<string, number> = new Map([
@@ -107,7 +107,7 @@ export class BurpSuiteMapper extends BaseConverter {
               },
               cci: {
                 path: 'vulnerabilityClassifications',
-                transformer: (data: string) => getCCIsForNISTTags(nistTag(data))
+                transformer: (data: string) => NIST2CCI(nistTag(data))
               },
               confidence: {path: 'confidence'}
             },

libs/hdf-converters/src/ckl-mapper/checklist-jsonix-converter.ts

@@ -1,7 +1,7 @@
 import {ExecJSON} from 'inspecjs';
 import _ from 'lodash';
 import {JsonixIntermediateConverter} from '../jsonix-intermediate-converter';
-import {CciNistTwoWayMapper} from '../mappings/CciNistMapping';
+import {NIST2CCI} from '../mappings/CciNistMapping';
 import {getDescription} from '../utils/global';
 import {
   Asset,
@@ -632,10 +632,9 @@ export class ChecklistJsonixConverter extends JsonixIntermediateConverter<
 
   matchNistToCcis(nistRefs: string[]): string[] {
     if (!nistRefs) {
-      return [''];
+      return [];
     }
-    const CCI_NIST_TWO_WAY_MAPPER = new CciNistTwoWayMapper();
-    return CCI_NIST_TWO_WAY_MAPPER.cciFilter(nistRefs, ['']);
+    return NIST2CCI(nistRefs);
   }
 
   getComments(descriptions: ExecJSON.ControlDescription[]): string {

libs/hdf-converters/src/ckl-mapper/checklist-mapper.ts

@@ -8,8 +8,7 @@ import {
   ILookupPath,
   MappedTransform
 } from '../base-converter';
-import {CciNistTwoWayMapper} from '../mappings/CciNistMapping';
-import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from '../utils/global';
+import {CCI2NIST} from '../mappings/CciNistMapping';
 import {
   ChecklistJsonixConverter,
   ChecklistObject,
@@ -21,15 +20,14 @@ import {Checklist} from './checklistJsonix';
 import {jsonixMapping} from './jsonixMapping';
 import {throwIfInvalidAssetMetadata} from './checklist-metadata-utils';
 import {parseJson} from '../utils/parseJson';
+import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from '../mappings/CciNistMappingData';
 
 enum ImpactMapping {
   high = 0.7,
   medium = 0.5,
   low = 0.3
 }
 
-const CCI_NIST_TWO_WAY_MAPPER = new CciNistTwoWayMapper();
-
 /**
  * Tranformer function that splits a string and return array
  * @param input - string of CCI references
@@ -47,10 +45,7 @@ function cciRef(input: string): string[] {
  */
 function nistTag(input: string): string[] {
   const identifiers: string[] = cciRef(input);
-  return CCI_NIST_TWO_WAY_MAPPER.nistFilter(
-    identifiers,
-    DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS
-  );
+  return CCI2NIST(identifiers, DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS);
 }
 
 /**

libs/hdf-converters/src/conveyor-mapper.ts

@@ -5,7 +5,7 @@ import {BaseConverter, ILookupPath, MappedTransform} from './base-converter';
 import {
   DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS,
   DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS
-} from './utils/global';
+} from './mappings/CciNistMappingData';
 const CONVEYOR_MAX_SCORE = 1000;
 enum scannerType {
   ClamAV = 'Clamav',
@@ -188,7 +188,7 @@ function controlMappingConveyor(): MappedTransform<
       size: {path: 'size'},
       type: {path: 'type'},
       nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS,
-      cci: DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS.flat()
+      cci: DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS
     },
     source_location: {},
     results: [

libs/hdf-converters/src/cyclonedx-sbom-mapper.ts

@@ -22,7 +22,7 @@ import {
   ComponentClass,
   ComponentObject
 } from '../types/cyclonedx';
-import {getCCIsForNISTTags} from './mappings/CciNistMapping';
+import {NIST2CCI} from './mappings/CciNistMapping';
 
 const cvssMethods = ['CVSSv2', 'CVSSv3', 'CVSSv31', 'CVSSv4'] as const;
 type CVSSMethodEnum = Extract<MethodEnum, (typeof cvssMethods)[number]>;
@@ -396,7 +396,7 @@ export class CycloneDXSBOMMapper extends BaseConverter<DataStorage> {
                   input:
                     | CycloneDXBillOfMaterialsStandardVulnerability['cwes']
                     | CycloneDXSoftwareBillOfMaterialsStandardVulnerability['cwes']
-                ): string[] => getCCIsForNISTTags(getNISTTags(input))
+                ): string[] => NIST2CCI(getNISTTags(input))
               },
               cwe: {path: 'cwes', transformer: formatCWETags},
               'bom-ref': {

libs/hdf-converters/src/dbprotect-mapper.ts

@@ -8,8 +8,8 @@ import {
   MappedTransform,
   parseXml
 } from './base-converter';
-import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from './utils/global';
-import {getCCIsForNISTTags} from './mappings/CciNistMapping';
+import {DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS} from './mappings/CciNistMappingData';
+import {NIST2CCI} from './mappings/CciNistMapping';
 
 const IMPACT_MAPPING: Map<string, number> = new Map([
   ['high', 0.7],
@@ -128,7 +128,7 @@ export class DBProtectMapper extends BaseConverter {
             key: 'id',
             tags: {
               nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS,
-              cci: getCCIsForNISTTags(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS)
+              cci: NIST2CCI(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS)
             },
             refs: [],
             source_location: {},