From 3ba96d55ca1f206b39c906107c4c0ef36c15801e Mon Sep 17 00:00:00 2001 From: Ryan Y Lin Date: Tue, 10 Aug 2021 08:51:55 -0700 Subject: [PATCH 01/10] Create aws-config-mapper.ts Bypassing merge conflicts Co-Authored-By: Rlin232 <38306648+Rlin232@users.noreply.github.com> --- libs/hdf-converters/src/aws-config-mapper.ts | 334 +++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 libs/hdf-converters/src/aws-config-mapper.ts diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts new file mode 100644 index 0000000000..ce4bf79310 --- /dev/null +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -0,0 +1,334 @@ +import { + ComplianceByConfigRule, + ConfigRule, + ConfigService, + ConfigServiceClientConfig, + DescribeConfigRulesCommandInput, + DescribeConfigRulesCommandOutput, + EvaluationResult +} from '@aws-sdk/client-config-service'; +import {ExecJSON} from 'inspecjs'; +import _ from 'lodash'; +import path from 'path'; +import {version as HeimdallToolsVersion} from '../package.json'; +import {AwsConfigMapping} from './mappings/AwsConfigMapping'; + +const NOT_APPLICABLE_MSG = + 'No AWS resources found to evaluate compliance for this rule'; +const INSUFFICIENT_DATA_MSG = + 'Not enough data has been collected to determine compliance yet.'; +const NAME = 'AWS Config'; + +const AWS_CONFIG_MAPPING_FILE = path.resolve( + __dirname, + '../data/aws-config-mapping.csv' +); +const AWS_CONFIG_MAPPING = new AwsConfigMapping(AWS_CONFIG_MAPPING_FILE); + +export class AwsConfigMapper { + configService: ConfigService; + issues: Promise; + results: ExecJSON.ControlResult[][]; + constructor(options: ConfigServiceClientConfig) { + this.configService = new ConfigService(options); + this.results = []; + this.issues = this.getAllConfigRules(); + } + private async getAllConfigRules(): Promise { + let params: DescribeConfigRulesCommandInput = { + ConfigRuleNames: [], + NextToken: '' + }; + const configRules: ConfigRule[] = []; + let response = await this.getConfigRulePage(params); + if (response.ConfigRules === undefined) { + throw new Error('No data was returned'); + } else { + while (response !== undefined && response.ConfigRules !== undefined) { + response.ConfigRules.forEach((rule) => { + configRules.push(rule); + }); + if (response.NextToken) { + params = _.set(params, 'NextToken', response.NextToken); + } else { + break; + } + response = await this.getConfigRulePage(params); + } + } + this.results = await this.getResults(configRules); + return configRules; + } + private async getConfigRulePage( + params: DescribeConfigRulesCommandInput + ): Promise { + return this.configService.describeConfigRules(params); + } + private async getResults( + configRules: ConfigRule[] + ): Promise { + const complianceResults: ComplianceByConfigRule[] = + await this.fetchAllComplianceInfo(configRules); + const results = configRules.map(async (rule) => { + const result: ExecJSON.ControlResult[] = []; + let params = { + ConfigRuleName: rule.ConfigRuleName, + Limit: 100 + }; + let response = await this.configService.getComplianceDetailsByConfigRule( + params + ); + let ruleResults = response.EvaluationResults || []; + while (response.NextToken !== undefined) { + params = _.set(params, 'NextToken', response.NextToken); + response = await this.configService.getComplianceDetailsByConfigRule( + params + ); + ruleResults = ruleResults?.concat(response.EvaluationResults || []); + } + ruleResults.forEach((evaluation) => { + const hdfResult: ExecJSON.ControlResult = { + code_desc: this.getCodeDesc(evaluation), + start_time: evaluation.ConfigRuleInvokedTime?.toISOString() || '', + run_time: this.getRunTime(evaluation), + status: this.getStatus(evaluation), + message: this.getMessage( + evaluation, + this.getCodeDesc(evaluation), + this.getStatus(evaluation) + ) + }; + result.push(hdfResult); + }); + const currentDate: string = new Date().toISOString(); + if (result.length === 0) { + switch ( + complianceResults.find( + (complianceResult) => + complianceResult.ConfigRuleName === rule.ConfigRuleName + )?.Compliance?.ComplianceType + ) { + case 'NOT_APPLICABLE': + return [ + { + run_time: 0, + code_desc: NOT_APPLICABLE_MSG, + skip_message: NOT_APPLICABLE_MSG, + start_time: currentDate, + status: ExecJSON.ControlResultStatus.Skipped + } + ]; + case 'INSUFFICIENT_DATA': + return [ + { + run_time: 0, + code_desc: INSUFFICIENT_DATA_MSG, + skip_message: INSUFFICIENT_DATA_MSG, + start_time: currentDate, + status: ExecJSON.ControlResultStatus.Skipped + } + ]; + default: + return []; + } + } else { + return result; + } + }); + const output: ExecJSON.ControlResult[][] = await Promise.all(results); + return output; + } + private getCodeDesc(result: EvaluationResult): string { + let output = ''; + if ( + result.EvaluationResultIdentifier !== undefined && + result.EvaluationResultIdentifier.EvaluationResultQualifier !== undefined + ) { + output = JSON.stringify( + result.EvaluationResultIdentifier.EvaluationResultQualifier + ) + .replace(/\"/gi, '') + .replace(/{/gi, '') + .replace(/}/gi, ''); + } + return output; + } + private getRunTime(result: EvaluationResult): number { + let diff = 0; + if ( + result.ResultRecordedTime !== undefined && + result.ConfigRuleInvokedTime !== undefined + ) { + diff = + (result.ResultRecordedTime.getTime() - + result.ConfigRuleInvokedTime.getTime()) / + 1000; + } + return diff; + } + private getStatus(result: EvaluationResult): ExecJSON.ControlResultStatus { + if (result.ComplianceType === 'COMPLIANT') { + return ExecJSON.ControlResultStatus.Passed; + } else if (result.ComplianceType === 'NON_COMPLIANT') { + return ExecJSON.ControlResultStatus.Failed; + } else { + return ExecJSON.ControlResultStatus.Skipped; + } + } + private getMessage( + result: EvaluationResult, + code_desc: string, + status: ExecJSON.ControlResultStatus + ): string | undefined { + if (status === ExecJSON.ControlResultStatus.Failed) { + return `${code_desc}: ${ + result.Annotation || 'Rule does not pass rule compliance' + }`; + } else { + return undefined; + } + } + private async fetchAllComplianceInfo( + configRules: ConfigRule[] + ): Promise { + const complianceResults: ComplianceByConfigRule[] = []; + for (const rule of configRules) { + const response = await this.configService.describeComplianceByConfigRule({ + ConfigRuleNames: [rule.ConfigRuleName || ''] + }); + if (response.ComplianceByConfigRules === undefined) { + throw new Error('No compliance data was returned'); + } else { + response.ComplianceByConfigRules?.forEach((compliance) => + complianceResults.push(compliance) + ); + } + } + return complianceResults; + } + // eslint-disable-next-line @typescript-eslint/ban-types + private hdfTags(configRule: ConfigRule): Record { + let result = {}; + const sourceIdentifier = configRule.Source?.SourceIdentifier; + result = _.set(result, 'nist', []); + let defaultMatch: string[] | null = []; + if (sourceIdentifier !== undefined) { + defaultMatch = AWS_CONFIG_MAPPING.nistFilter([sourceIdentifier]); + } + if (Array.isArray(defaultMatch) && defaultMatch.length !== 0) { + result = _.set( + result, + 'nist', + _.get(result, 'nist').concat(defaultMatch) + ); + } + if ( + Array.isArray(_.get(result, 'nist')) && + _.get(result, 'nist').length === 0 + ) { + result = _.set(result, 'nist', ['unmapped']); + } + return result; + } + private checkText(configRule: ConfigRule): string { + let params: any[] = []; + if ( + configRule.InputParameters !== undefined && + configRule.InputParameters !== '{}' + ) { + params = configRule.InputParameters.replace(/{/gi, '') + .replace(/}/gi, '') + .split(','); + } + const checkText = []; + checkText.push(`ARN: ${configRule.ConfigRuleArn || 'N/A'}`); + checkText.push( + `Source Identifier: ${configRule.Source?.SourceIdentifier || 'N/A'}` + ); + if (params.length !== 0) { + checkText.push(`${params.join('
').replace(/\"/gi, '')}`); + } + return checkText.join('
'); + } + private hdfDescriptions(configRule: ConfigRule) { + return [ + { + data: this.checkText(configRule), + label: 'check' + } + ]; + } + private getAccountId(arn: string): string { + const matches = arn.match(/:(\d{12}):config-rule/); + if (matches === null) { + return 'no-account-id'; + } else { + return matches[0]; + } + } + private async getControls(): Promise { + let index = 0; + const controls = (await this.issues).map((issue: ConfigRule) => { + const control: ExecJSON.Control = { + id: issue.ConfigRuleId || '', + title: `${this.getAccountId(issue.ConfigRuleArn || '')} - ${ + issue.ConfigRuleName + }` + .replace(/:/gi, '') + .replace(/config-rule/gi, ''), + desc: issue.Description || null, + impact: this.getImpact(issue), + tags: this.hdfTags(issue), + descriptions: this.hdfDescriptions(issue), + refs: [], + source_location: {ref: issue.ConfigRuleArn, line: 1}, + code: '', + results: this.results[index] + }; + index++; + return control; + }); + return controls; + } + private getImpact(issue: ConfigRule): number { + if (_.get(issue, 'compliance') === 'NOT_APPLICABLE') { + return 0; + } else { + return 0.5; + } + } + public async toHdf(): Promise { + const hdf: ExecJSON.Execution = { + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id: '' + }, + version: HeimdallToolsVersion, + statistics: { + //aws_config_sdk_version: ConfigService., // How do i get the sdk version? + duration: null + }, + profiles: [ + { + name: NAME, + version: '', + title: NAME, + maintainer: null, + summary: NAME, + license: null, + copyright: null, + copyright_email: null, + supports: [], + attributes: [], + depends: [], + groups: [], + status: 'loaded', + controls: await this.getControls(), + sha256: '' + } + ] + }; + return hdf; + } +} From 0c0aa0316b2d47c983b535c5759ae5d2e6915d4f Mon Sep 17 00:00:00 2001 From: Camden Moors Date: Mon, 13 Dec 2021 22:47:50 +0300 Subject: [PATCH 02/10] Add delays, fix AWS Config Mappings Import --- libs/hdf-converters/src/aws-config-mapper.ts | 36 ++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index ce4bf79310..6d79d05af4 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -9,7 +9,6 @@ import { } from '@aws-sdk/client-config-service'; import {ExecJSON} from 'inspecjs'; import _ from 'lodash'; -import path from 'path'; import {version as HeimdallToolsVersion} from '../package.json'; import {AwsConfigMapping} from './mappings/AwsConfigMapping'; @@ -19,11 +18,7 @@ const INSUFFICIENT_DATA_MSG = 'Not enough data has been collected to determine compliance yet.'; const NAME = 'AWS Config'; -const AWS_CONFIG_MAPPING_FILE = path.resolve( - __dirname, - '../data/aws-config-mapping.csv' -); -const AWS_CONFIG_MAPPING = new AwsConfigMapping(AWS_CONFIG_MAPPING_FILE); +const AWS_CONFIG_MAPPING = new AwsConfigMapping(); export class AwsConfigMapper { configService: ConfigService; @@ -34,6 +29,11 @@ export class AwsConfigMapper { this.results = []; this.issues = this.getAllConfigRules(); } + + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + private async getAllConfigRules(): Promise { let params: DescribeConfigRulesCommandInput = { ConfigRuleNames: [], @@ -59,11 +59,14 @@ export class AwsConfigMapper { this.results = await this.getResults(configRules); return configRules; } + private async getConfigRulePage( params: DescribeConfigRulesCommandInput ): Promise { + await this.delay(500); return this.configService.describeConfigRules(params); } + private async getResults( configRules: ConfigRule[] ): Promise { @@ -75,12 +78,14 @@ export class AwsConfigMapper { ConfigRuleName: rule.ConfigRuleName, Limit: 100 }; + await this.delay(500); let response = await this.configService.getComplianceDetailsByConfigRule( params ); let ruleResults = response.EvaluationResults || []; while (response.NextToken !== undefined) { params = _.set(params, 'NextToken', response.NextToken); + await this.delay(500); response = await this.configService.getComplianceDetailsByConfigRule( params ); @@ -138,6 +143,7 @@ export class AwsConfigMapper { const output: ExecJSON.ControlResult[][] = await Promise.all(results); return output; } + private getCodeDesc(result: EvaluationResult): string { let output = ''; if ( @@ -153,6 +159,7 @@ export class AwsConfigMapper { } return output; } + private getRunTime(result: EvaluationResult): number { let diff = 0; if ( @@ -166,6 +173,7 @@ export class AwsConfigMapper { } return diff; } + private getStatus(result: EvaluationResult): ExecJSON.ControlResultStatus { if (result.ComplianceType === 'COMPLIANT') { return ExecJSON.ControlResultStatus.Passed; @@ -175,19 +183,21 @@ export class AwsConfigMapper { return ExecJSON.ControlResultStatus.Skipped; } } + private getMessage( result: EvaluationResult, - code_desc: string, + codeDesc: string, status: ExecJSON.ControlResultStatus ): string | undefined { if (status === ExecJSON.ControlResultStatus.Failed) { - return `${code_desc}: ${ + return `${codeDesc}: ${ result.Annotation || 'Rule does not pass rule compliance' }`; } else { return undefined; } } + private async fetchAllComplianceInfo( configRules: ConfigRule[] ): Promise { @@ -206,6 +216,7 @@ export class AwsConfigMapper { } return complianceResults; } + // eslint-disable-next-line @typescript-eslint/ban-types private hdfTags(configRule: ConfigRule): Record { let result = {}; @@ -230,6 +241,7 @@ export class AwsConfigMapper { } return result; } + private checkText(configRule: ConfigRule): string { let params: any[] = []; if ( @@ -250,6 +262,7 @@ export class AwsConfigMapper { } return checkText.join('
'); } + private hdfDescriptions(configRule: ConfigRule) { return [ { @@ -258,6 +271,7 @@ export class AwsConfigMapper { } ]; } + private getAccountId(arn: string): string { const matches = arn.match(/:(\d{12}):config-rule/); if (matches === null) { @@ -266,9 +280,10 @@ export class AwsConfigMapper { return matches[0]; } } + private async getControls(): Promise { let index = 0; - const controls = (await this.issues).map((issue: ConfigRule) => { + return (await this.issues).map((issue: ConfigRule) => { const control: ExecJSON.Control = { id: issue.ConfigRuleId || '', title: `${this.getAccountId(issue.ConfigRuleArn || '')} - ${ @@ -288,8 +303,8 @@ export class AwsConfigMapper { index++; return control; }); - return controls; } + private getImpact(issue: ConfigRule): number { if (_.get(issue, 'compliance') === 'NOT_APPLICABLE') { return 0; @@ -297,6 +312,7 @@ export class AwsConfigMapper { return 0.5; } } + public async toHdf(): Promise { const hdf: ExecJSON.Execution = { platform: { From abe0c834a5b0f70a36d62e7b4c78a224c0e333f6 Mon Sep 17 00:00:00 2001 From: Camden Moors Date: Mon, 13 Dec 2021 23:47:16 +0300 Subject: [PATCH 03/10] Wip add tests --- libs/hdf-converters/test/hdf_checker.spec.ts | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/libs/hdf-converters/test/hdf_checker.spec.ts b/libs/hdf-converters/test/hdf_checker.spec.ts index 1b6326b12a..c9d23f8628 100644 --- a/libs/hdf-converters/test/hdf_checker.spec.ts +++ b/libs/hdf-converters/test/hdf_checker.spec.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import {ExecJSON} from 'inspecjs'; import _ from 'lodash'; import {ASFFMapper} from '../src/asff-mapper'; +import {AwsConfigMapper} from '../src/aws-config-mapper'; import {BurpSuiteMapper} from '../src/burpsuite-mapper'; import {IFindingASFF} from '../src/converters-from-hdf/asff/asff-types'; import {FromHdfToAsffMapper} from '../src/converters-from-hdf/asff/reverse-asff-mapper'; @@ -67,7 +68,7 @@ test('Test converter toASFF function', () => { expect(omitASFFTimes(converted)).toEqual(omitASFFTimes(expectedJSON)); }); -describe('asff_mapper', () => { +describe('Test asff_mapper', () => { it('Successfully converts Native ASFF', () => { const mapper = new ASFFMapper( fs.readFileSync( @@ -124,8 +125,12 @@ describe('asff_mapper', () => { ); }); }); - -test('Test burpsuite_mapper', () => { +test('aws_config_mapper', async () => { + const mapper = new AwsConfigMapper({region: 'ca-central-1'}); + const data = await mapper.toHdf(); + console.log(JSON.stringify(data)); +}); +test('burpsuite_mapper', () => { const mapper = new BurpSuiteMapper( fs.readFileSync( 'sample_jsons/burpsuite_mapper/sample_input_report/zero.webappsecurity.com.min', @@ -142,8 +147,7 @@ test('Test burpsuite_mapper', () => { ) ); }); - -test('Test jfrog_xray_mapper', () => { +test('jfrog_xray_mapper', () => { const mapper = new JfrogXrayMapper( fs.readFileSync( 'sample_jsons/jfrog_xray_mapper/sample_input_report/jfrog_xray_sample.json', @@ -160,7 +164,7 @@ test('Test jfrog_xray_mapper', () => { ) ); }); -test('Test nikto_mapper', () => { +test('nikto_mapper', () => { const mapper = new NiktoMapper( fs.readFileSync( 'sample_jsons/nikto_mapper/sample_input_report/zero.webappsecurity.json', @@ -177,7 +181,7 @@ test('Test nikto_mapper', () => { ) ); }); -test('Test sarif_mapper', () => { +test('sarif_mapper', () => { const mapper = new SarifMapper( fs.readFileSync( 'sample_jsons/sarif_mapper/sample_input_report/sarif_input.sarif', @@ -194,7 +198,7 @@ test('Test sarif_mapper', () => { ) ); }); -test('Test scoutsuite_mapper', () => { +test('scoutsuite_mapper', () => { const mapper = new ScoutsuiteMapper( fs.readFileSync( 'sample_jsons/scoutsuite_mapper/sample_input_report/scoutsuite_sample.js', @@ -213,7 +217,7 @@ test('Test scoutsuite_mapper', () => { ) ); }); -test('Test sonarqube_mapper', async () => { +test('sonarqube_mapper', async () => { const mapper = new SonarQubeResults( 'http://127.0.0.1:3001', 'xss', @@ -230,8 +234,7 @@ test('Test sonarqube_mapper', async () => { ) ); }); - -test('Test xccdf_results_mapper', () => { +test('xccdf_results_mapper', () => { const mapper = new XCCDFResultsMapper( fs.readFileSync( 'sample_jsons/xccdf_results_mapper/sample_input_report/xccdf-results.xml', @@ -248,7 +251,7 @@ test('Test xccdf_results_mapper', () => { ) ); }); -test('Test zap_mapper webgoat.json', () => { +test('zap_mapper webgoat.json', () => { const mapper = new ZapMapper( fs.readFileSync( 'sample_jsons/zap_mapper/sample_input_report/webgoat.json', @@ -266,7 +269,7 @@ test('Test zap_mapper webgoat.json', () => { ) ); }); -test('Test zap_mapper zero.webappsecurity.json', () => { +test('zap_mapper zero.webappsecurity.json', () => { const mapper = new ZapMapper( fs.readFileSync( 'sample_jsons/zap_mapper/sample_input_report/zero.webappsecurity.json', From c5a28289ad16c0fe09101f55fed9c279dad942cd Mon Sep 17 00:00:00 2001 From: Camden Moors Date: Tue, 14 Dec 2021 19:55:19 +0300 Subject: [PATCH 04/10] Fix ratelimiting for getting rule results --- libs/hdf-converters/src/aws-config-mapper.ts | 100 +++++++++++-------- 1 file changed, 56 insertions(+), 44 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index 6d79d05af4..7d56f175dc 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -60,10 +60,18 @@ export class AwsConfigMapper { return configRules; } + private chunkArray(sourceArray: Array, chunkSize: number) { + const result = []; + for (let i = 0; i < sourceArray.length; i += chunkSize) { + result.push(sourceArray.slice(i, i + chunkSize)); + } + return result; + } + private async getConfigRulePage( params: DescribeConfigRulesCommandInput ): Promise { - await this.delay(500); + await this.delay(5000); return this.configService.describeConfigRules(params); } @@ -72,20 +80,21 @@ export class AwsConfigMapper { ): Promise { const complianceResults: ComplianceByConfigRule[] = await this.fetchAllComplianceInfo(configRules); - const results = configRules.map(async (rule) => { + const ruleData: ExecJSON.ControlResult[][] = []; + for (let i = 0; i < configRules.length; i++) { const result: ExecJSON.ControlResult[] = []; let params = { - ConfigRuleName: rule.ConfigRuleName, + ConfigRuleName: configRules[i].ConfigRuleName, Limit: 100 }; - await this.delay(500); + await this.delay(750); let response = await this.configService.getComplianceDetailsByConfigRule( params ); let ruleResults = response.EvaluationResults || []; while (response.NextToken !== undefined) { params = _.set(params, 'NextToken', response.NextToken); - await this.delay(500); + await this.delay(5000); response = await this.configService.getComplianceDetailsByConfigRule( params ); @@ -104,44 +113,44 @@ export class AwsConfigMapper { ) }; result.push(hdfResult); - }); - const currentDate: string = new Date().toISOString(); - if (result.length === 0) { - switch ( - complianceResults.find( - (complianceResult) => - complianceResult.ConfigRuleName === rule.ConfigRuleName - )?.Compliance?.ComplianceType - ) { - case 'NOT_APPLICABLE': - return [ - { - run_time: 0, - code_desc: NOT_APPLICABLE_MSG, - skip_message: NOT_APPLICABLE_MSG, - start_time: currentDate, - status: ExecJSON.ControlResultStatus.Skipped - } - ]; - case 'INSUFFICIENT_DATA': - return [ - { - run_time: 0, - code_desc: INSUFFICIENT_DATA_MSG, - skip_message: INSUFFICIENT_DATA_MSG, - start_time: currentDate, - status: ExecJSON.ControlResultStatus.Skipped - } - ]; - default: - return []; + const currentDate: string = new Date().toISOString(); + if (result.length === 0) { + switch ( + complianceResults.find( + (complianceResult) => + complianceResult.ConfigRuleName === + configRules[i].ConfigRuleName + )?.Compliance?.ComplianceType + ) { + case 'NOT_APPLICABLE': + return [ + { + run_time: 0, + code_desc: NOT_APPLICABLE_MSG, + skip_message: NOT_APPLICABLE_MSG, + start_time: currentDate, + status: ExecJSON.ControlResultStatus.Skipped + } + ]; + case 'INSUFFICIENT_DATA': + return [ + { + run_time: 0, + code_desc: INSUFFICIENT_DATA_MSG, + skip_message: INSUFFICIENT_DATA_MSG, + start_time: currentDate, + status: ExecJSON.ControlResultStatus.Skipped + } + ]; + default: + return []; + } + } else { + return ruleData.push(result); } - } else { - return result; - } - }); - const output: ExecJSON.ControlResult[][] = await Promise.all(results); - return output; + }); + } + return await Promise.all(ruleData); } private getCodeDesc(result: EvaluationResult): string { @@ -202,9 +211,12 @@ export class AwsConfigMapper { configRules: ConfigRule[] ): Promise { const complianceResults: ComplianceByConfigRule[] = []; - for (const rule of configRules) { + // Should slice config rules into arrays of max size: 25 and make one request for each slice + const configRuleSlices = this.chunkArray(configRules, 25); + for (const slice of configRuleSlices) { + await this.delay(5000); const response = await this.configService.describeComplianceByConfigRule({ - ConfigRuleNames: [rule.ConfigRuleName || ''] + ConfigRuleNames: slice.map((rule) => rule.ConfigRuleName || '') }); if (response.ComplianceByConfigRules === undefined) { throw new Error('No compliance data was returned'); From 01cc64b634dc2d9895b4ed07c9e07bc34c0ea663 Mon Sep 17 00:00:00 2001 From: Camden Moors Date: Tue, 14 Dec 2021 20:03:11 +0300 Subject: [PATCH 05/10] Restore aws-sdk import change --- libs/hdf-converters/src/aws-config-mapper.ts | 49 ++++++++++++-------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index 7d56f175dc..ba8582d68b 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -1,12 +1,11 @@ import { ComplianceByConfigRule, ConfigRule, - ConfigService, - ConfigServiceClientConfig, DescribeConfigRulesCommandInput, - DescribeConfigRulesCommandOutput, EvaluationResult } from '@aws-sdk/client-config-service'; +import AWS from 'aws-sdk'; +import https from 'https'; import {ExecJSON} from 'inspecjs'; import _ from 'lodash'; import {version as HeimdallToolsVersion} from '../package.json'; @@ -21,11 +20,21 @@ const NAME = 'AWS Config'; const AWS_CONFIG_MAPPING = new AwsConfigMapping(); export class AwsConfigMapper { - configService: ConfigService; + configService: AWS.ConfigService; issues: Promise; results: ExecJSON.ControlResult[][]; - constructor(options: ConfigServiceClientConfig) { - this.configService = new ConfigService(options); + constructor( + options: AWS.ConfigService.ClientConfiguration, + verifySSLCertificates = true + ) { + AWS.config.update({ + httpOptions: { + agent: new https.Agent({ + rejectUnauthorized: verifySSLCertificates + }) + } + }); + this.configService = new AWS.ConfigService(options); this.results = []; this.issues = this.getAllConfigRules(); } @@ -70,9 +79,9 @@ export class AwsConfigMapper { private async getConfigRulePage( params: DescribeConfigRulesCommandInput - ): Promise { - await this.delay(5000); - return this.configService.describeConfigRules(params); + ): Promise { + await this.delay(500); + return this.configService.describeConfigRules(params).promise(); } private async getResults( @@ -84,20 +93,20 @@ export class AwsConfigMapper { for (let i = 0; i < configRules.length; i++) { const result: ExecJSON.ControlResult[] = []; let params = { - ConfigRuleName: configRules[i].ConfigRuleName, + ConfigRuleName: configRules[i].ConfigRuleName || '', Limit: 100 }; await this.delay(750); - let response = await this.configService.getComplianceDetailsByConfigRule( - params - ); + let response = await this.configService + .getComplianceDetailsByConfigRule(params) + .promise(); let ruleResults = response.EvaluationResults || []; while (response.NextToken !== undefined) { params = _.set(params, 'NextToken', response.NextToken); await this.delay(5000); - response = await this.configService.getComplianceDetailsByConfigRule( - params - ); + response = await this.configService + .getComplianceDetailsByConfigRule(params) + .promise(); ruleResults = ruleResults?.concat(response.EvaluationResults || []); } ruleResults.forEach((evaluation) => { @@ -215,9 +224,11 @@ export class AwsConfigMapper { const configRuleSlices = this.chunkArray(configRules, 25); for (const slice of configRuleSlices) { await this.delay(5000); - const response = await this.configService.describeComplianceByConfigRule({ - ConfigRuleNames: slice.map((rule) => rule.ConfigRuleName || '') - }); + const response = await this.configService + .describeComplianceByConfigRule({ + ConfigRuleNames: slice.map((rule) => rule.ConfigRuleName || '') + }) + .promise(); if (response.ComplianceByConfigRules === undefined) { throw new Error('No compliance data was returned'); } else { From b2a2287802f6623b8e36493450a53d4ae45e5ef1 Mon Sep 17 00:00:00 2001 From: Camden Moors Date: Tue, 14 Dec 2021 20:56:31 +0300 Subject: [PATCH 06/10] Update delay times, switch to for of --- libs/hdf-converters/src/aws-config-mapper.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index ba8582d68b..f19d90d064 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -80,7 +80,7 @@ export class AwsConfigMapper { private async getConfigRulePage( params: DescribeConfigRulesCommandInput ): Promise { - await this.delay(500); + await this.delay(10); return this.configService.describeConfigRules(params).promise(); } @@ -90,20 +90,20 @@ export class AwsConfigMapper { const complianceResults: ComplianceByConfigRule[] = await this.fetchAllComplianceInfo(configRules); const ruleData: ExecJSON.ControlResult[][] = []; - for (let i = 0; i < configRules.length; i++) { + for (const configRule of configRules) { const result: ExecJSON.ControlResult[] = []; let params = { - ConfigRuleName: configRules[i].ConfigRuleName || '', + ConfigRuleName: configRule.ConfigRuleName || '', Limit: 100 }; - await this.delay(750); + await this.delay(10); let response = await this.configService .getComplianceDetailsByConfigRule(params) .promise(); let ruleResults = response.EvaluationResults || []; while (response.NextToken !== undefined) { params = _.set(params, 'NextToken', response.NextToken); - await this.delay(5000); + await this.delay(100); response = await this.configService .getComplianceDetailsByConfigRule(params) .promise(); @@ -127,8 +127,7 @@ export class AwsConfigMapper { switch ( complianceResults.find( (complianceResult) => - complianceResult.ConfigRuleName === - configRules[i].ConfigRuleName + complianceResult.ConfigRuleName === configRule.ConfigRuleName )?.Compliance?.ComplianceType ) { case 'NOT_APPLICABLE': @@ -159,7 +158,7 @@ export class AwsConfigMapper { } }); } - return await Promise.all(ruleData); + return Promise.all(ruleData); } private getCodeDesc(result: EvaluationResult): string { @@ -223,7 +222,7 @@ export class AwsConfigMapper { // Should slice config rules into arrays of max size: 25 and make one request for each slice const configRuleSlices = this.chunkArray(configRules, 25); for (const slice of configRuleSlices) { - await this.delay(5000); + await this.delay(100); const response = await this.configService .describeComplianceByConfigRule({ ConfigRuleNames: slice.map((rule) => rule.ConfigRuleName || '') From b1b41789603a83a3c028b472236b9e486bcadc41 Mon Sep 17 00:00:00 2001 From: Camden Moors <66680985+camdenmoors@users.noreply.github.com> Date: Wed, 15 Dec 2021 15:51:52 -0500 Subject: [PATCH 07/10] Append resource IDs to code_desc if match is found --- libs/hdf-converters/src/aws-config-mapper.ts | 89 ++++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index f19d90d064..384c952d42 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -80,7 +80,7 @@ export class AwsConfigMapper { private async getConfigRulePage( params: DescribeConfigRulesCommandInput ): Promise { - await this.delay(10); + await this.delay(150); return this.configService.describeConfigRules(params).promise(); } @@ -90,24 +90,27 @@ export class AwsConfigMapper { const complianceResults: ComplianceByConfigRule[] = await this.fetchAllComplianceInfo(configRules); const ruleData: ExecJSON.ControlResult[][] = []; + const allRulesResolved: AWS.ConfigService.EvaluationResults = []; for (const configRule of configRules) { const result: ExecJSON.ControlResult[] = []; let params = { ConfigRuleName: configRule.ConfigRuleName || '', Limit: 100 }; - await this.delay(10); + await this.delay(150); let response = await this.configService .getComplianceDetailsByConfigRule(params) .promise(); let ruleResults = response.EvaluationResults || []; + allRulesResolved.push(...ruleResults); while (response.NextToken !== undefined) { params = _.set(params, 'NextToken', response.NextToken); - await this.delay(100); + await this.delay(150); response = await this.configService .getComplianceDetailsByConfigRule(params) .promise(); ruleResults = ruleResults?.concat(response.EvaluationResults || []); + allRulesResolved.push(...ruleResults); } ruleResults.forEach((evaluation) => { const hdfResult: ExecJSON.ControlResult = { @@ -158,7 +161,83 @@ export class AwsConfigMapper { } }); } - return Promise.all(ruleData); + const completedControlResults = await Promise.all(ruleData); + const extractedResourceNames = await this.extractResourceNamesFromIds( + allRulesResolved + ); + + return completedControlResults.map((completedControlResult) => + completedControlResult.map((completedControl) => { + for (const extractedResourceName in extractedResourceNames) { + if ( + completedControl.code_desc.indexOf( + JSON.stringify(extractedResourceName) + .replace(/\"/gi, '') + .replace(/{/gi, '') + .replace(/}/gi, '') + ) !== -1 + ) { + return { + ...completedControl, + code_desc: + completedControl.code_desc + + ', resource_name:' + + extractedResourceNames[extractedResourceName] + }; + } + } + return completedControl; + }) + ); + } + + private async extractResourceNamesFromIds( + evaluationResults: AWS.ConfigService.EvaluationResults + ) { + // Map of resource types to resource IDs {resourceType: ResourceId[]} + const resourceMap: Record = {}; + // Map of resource IDs to resource names + const resolvedResourcesMap: Record = {}; + // Extract resource Ids + evaluationResults.forEach((result) => { + const resourceType: string = _.get( + result, + 'EvaluationResultIdentifier.EvaluationResultQualifier.ResourceType' + ); + const resourceId: string = _.get( + result, + 'EvaluationResultIdentifier.EvaluationResultQualifier.ResourceId' + ); + if (!(resourceType in resourceMap)) { + resourceMap[resourceType] = [resourceId]; + } else { + if ( + !resourceMap[resourceType].includes(resourceId) && + typeof resourceId === 'string' + ) { + resourceMap[resourceType].push(resourceId); + } + } + }); + // Resolve resource names from AWS + for (const resourceType in resourceMap) { + const resourceIDSlices = this.chunkArray(resourceMap[resourceType], 20); + for (const slice of resourceIDSlices) { + await this.delay(150); + const resources = await this.configService + .listDiscoveredResources({ + resourceType: resourceType, + resourceIds: slice + }) + .promise(); + resources.resourceIdentifiers?.forEach((resource) => { + if (resource.resourceId && resource.resourceName) { + resolvedResourcesMap[resource.resourceId] = resource.resourceName; + } + }); + } + } + return resolvedResourcesMap; } private getCodeDesc(result: EvaluationResult): string { @@ -222,7 +301,7 @@ export class AwsConfigMapper { // Should slice config rules into arrays of max size: 25 and make one request for each slice const configRuleSlices = this.chunkArray(configRules, 25); for (const slice of configRuleSlices) { - await this.delay(100); + await this.delay(150); const response = await this.configService .describeComplianceByConfigRule({ ConfigRuleNames: slice.map((rule) => rule.ConfigRuleName || '') From e6b6eda6f9afe4dc4c0e870e8fffc392640bc1d5 Mon Sep 17 00:00:00 2001 From: Camden Moors <66680985+camdenmoors@users.noreply.github.com> Date: Wed, 15 Dec 2021 15:54:32 -0500 Subject: [PATCH 08/10] Remove HDF Dump AWS Config test --- libs/hdf-converters/test/hdf_checker.spec.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/libs/hdf-converters/test/hdf_checker.spec.ts b/libs/hdf-converters/test/hdf_checker.spec.ts index c9d23f8628..b0cbe542a2 100644 --- a/libs/hdf-converters/test/hdf_checker.spec.ts +++ b/libs/hdf-converters/test/hdf_checker.spec.ts @@ -2,7 +2,6 @@ import fs from 'fs'; import {ExecJSON} from 'inspecjs'; import _ from 'lodash'; import {ASFFMapper} from '../src/asff-mapper'; -import {AwsConfigMapper} from '../src/aws-config-mapper'; import {BurpSuiteMapper} from '../src/burpsuite-mapper'; import {IFindingASFF} from '../src/converters-from-hdf/asff/asff-types'; import {FromHdfToAsffMapper} from '../src/converters-from-hdf/asff/reverse-asff-mapper'; @@ -125,11 +124,7 @@ describe('Test asff_mapper', () => { ); }); }); -test('aws_config_mapper', async () => { - const mapper = new AwsConfigMapper({region: 'ca-central-1'}); - const data = await mapper.toHdf(); - console.log(JSON.stringify(data)); -}); + test('burpsuite_mapper', () => { const mapper = new BurpSuiteMapper( fs.readFileSync( @@ -147,6 +142,7 @@ test('burpsuite_mapper', () => { ) ); }); + test('jfrog_xray_mapper', () => { const mapper = new JfrogXrayMapper( fs.readFileSync( @@ -164,6 +160,7 @@ test('jfrog_xray_mapper', () => { ) ); }); + test('nikto_mapper', () => { const mapper = new NiktoMapper( fs.readFileSync( @@ -181,6 +178,7 @@ test('nikto_mapper', () => { ) ); }); + test('sarif_mapper', () => { const mapper = new SarifMapper( fs.readFileSync( From 07f2902f6d897776fe9ed5e59391f9d424f74805 Mon Sep 17 00:00:00 2001 From: Camden Moors <66680985+camdenmoors@users.noreply.github.com> Date: Wed, 15 Dec 2021 16:01:57 -0500 Subject: [PATCH 09/10] Remove string concatenation, break appendResourceNamesToResults into a separate function --- libs/hdf-converters/src/aws-config-mapper.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/libs/hdf-converters/src/aws-config-mapper.ts b/libs/hdf-converters/src/aws-config-mapper.ts index 384c952d42..ada4fb8135 100644 --- a/libs/hdf-converters/src/aws-config-mapper.ts +++ b/libs/hdf-converters/src/aws-config-mapper.ts @@ -161,11 +161,17 @@ export class AwsConfigMapper { } }); } - const completedControlResults = await Promise.all(ruleData); - const extractedResourceNames = await this.extractResourceNamesFromIds( - allRulesResolved + + return this.appendResourceNamesToResults( + await Promise.all(ruleData), + await this.extractResourceNamesFromIds(allRulesResolved) ); + } + private async appendResourceNamesToResults( + completedControlResults: ExecJSON.ControlResult[][], + extractedResourceNames: Record + ) { return completedControlResults.map((completedControlResult) => completedControlResult.map((completedControl) => { for (const extractedResourceName in extractedResourceNames) { @@ -179,10 +185,7 @@ export class AwsConfigMapper { ) { return { ...completedControl, - code_desc: - completedControl.code_desc + - ', resource_name:' + - extractedResourceNames[extractedResourceName] + code_desc: `${completedControl.code_desc}, resource_name: ${extractedResourceNames[extractedResourceName]}` }; } } From 0fcbd1335fee5e7a17268ec9ec3a6ef219fdab21 Mon Sep 17 00:00:00 2001 From: Camden Moors <66680985+camdenmoors@users.noreply.github.com> Date: Wed, 15 Dec 2021 16:05:24 -0500 Subject: [PATCH 10/10] Include aws-sdk in hdf-converters --- libs/hdf-converters/package.json | 1 + yarn.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/hdf-converters/package.json b/libs/hdf-converters/package.json index 7feb46be28..cc4735324e 100644 --- a/libs/hdf-converters/package.json +++ b/libs/hdf-converters/package.json @@ -23,6 +23,7 @@ "dependencies": { "@types/csv2json": "^1.4.2", "@types/xml2js": "^0.4.9", + "aws-sdk": "^2.1046.0", "axios": "^0.24.0", "csv-parse": "^5.0.1", "csv2json": "^2.0.2", diff --git a/yarn.lock b/yarn.lock index 4df787bd52..dd4c887e16 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5855,7 +5855,7 @@ autoprefixer@^9.8.6: postcss "^7.0.32" postcss-value-parser "^4.1.0" -aws-sdk@^2.573.0: +aws-sdk@^2.1046.0, aws-sdk@^2.573.0: version "2.1046.0" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1046.0.tgz#9147b0fa1c86acbebd1a061e951ab5012f4499d7" integrity sha512-ocwHclMXdIA+NWocUyvp9Ild3/zy2vr5mHp3mTyodf0WU5lzBE8PocCVLSWhMAXLxyia83xv2y5f5AzAcetbqA==