diff --git a/src/commands/prepareAnalyzeCommand.ts b/src/commands/prepareAnalyzeCommand.ts index bc61726..60e575e 100644 --- a/src/commands/prepareAnalyzeCommand.ts +++ b/src/commands/prepareAnalyzeCommand.ts @@ -1,4 +1,5 @@ import { Logger } from "../cli/Logger.ts"; +import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts"; import { FoundationDependencies, KitDependencyAnalyzer, @@ -32,11 +33,22 @@ async function analyze( const validator = new ModelValidator(logger); const modules = await KitModuleRepository.load(collie, validator, logger); + const controls = await ComplianceControlRepository.load( + collie, + validator, + logger, + ); + const foundations = await collie.listFoundations(); const tasks = foundations.map(async (f) => { const foundation = await FoundationRepository.load(collie, f, validator); - const analyzer = new KitDependencyAnalyzer(collie, modules, logger); + const analyzer = new KitDependencyAnalyzer( + collie, + modules, + controls, + logger, + ); return { foundation, diff --git a/src/docs/ComplianceDocumentationGenerator.ts b/src/docs/ComplianceDocumentationGenerator.ts deleted file mode 100644 index 7148bcb..0000000 --- a/src/docs/ComplianceDocumentationGenerator.ts +++ /dev/null @@ -1,28 +0,0 @@ -import * as fs from "std/fs"; -import * as path from "std/path"; -import { Logger } from "../cli/Logger.ts"; -import { CollieRepository } from "../model/CollieRepository.ts"; -import { DocumentationRepository } from "./DocumentationRepository.ts"; - -export class ComplianceDocumentationGenerator { - constructor( - private readonly collie: CollieRepository, - private readonly logger: Logger, - ) {} - - public async generate(docsRepo: DocumentationRepository) { - const source = this.collie.resolvePath("compliance"); - - const destinationDir = docsRepo.resolveCompliancePath(); - this.logger.verbose( - (fmt) => - `copying (recursive) ${fmt.kitPath(source)} to ${ - fmt.kitPath( - destinationDir, - ) - }`, - ); - await Deno.mkdir(path.dirname(destinationDir), { recursive: true }); - await fs.copy(source, destinationDir, { overwrite: true }); - } -} diff --git a/src/docs/DocumentationGenerator.ts b/src/docs/DocumentationGenerator.ts deleted file mode 100644 index 04e1b41..0000000 --- a/src/docs/DocumentationGenerator.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { ComplianceDocumentationGenerator } from "./ComplianceDocumentationGenerator.ts"; -import { DocumentationRepository } from "./DocumentationRepository.ts"; -import { KitModuleDocumentationGenerator } from "./KitModuleDocumentationGenerator.ts"; -import { PlatformDocumentationGenerator } from "./PlatformDocumentationGenerator.ts"; - -export class DocumentationGenerator { - constructor( - private readonly kitModuleDocumentation: KitModuleDocumentationGenerator, - private readonly complianceDocumentation: ComplianceDocumentationGenerator, - private readonly platformDocumentation: PlatformDocumentationGenerator, - ) {} - - async generateFoundationDocumentation(docsRepo: DocumentationRepository) { - // todo: can we flatten the duplicate docs/ folder nesting? - await this.complianceDocumentation.generate(docsRepo); - await this.kitModuleDocumentation.generate(docsRepo); - await this.platformDocumentation.generate(docsRepo); - } -} diff --git a/src/docs/KitModuleDocumentationGenerator.ts b/src/docs/KitModuleDocumentationGenerator.ts deleted file mode 100644 index c101c9c..0000000 --- a/src/docs/KitModuleDocumentationGenerator.ts +++ /dev/null @@ -1,102 +0,0 @@ -import * as fs from "std/fs"; -import * as path from "std/path"; - -import { Logger } from "../cli/Logger.ts"; -import { ProgressReporter } from "../cli/ProgressReporter.ts"; -import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts"; -import { KitModuleRepository } from "../kit/KitModuleRepository.ts"; -import { ParsedKitModule } from "../kit/ParsedKitModule.ts"; -import { CollieRepository } from "../model/CollieRepository.ts"; -import { DocumentationRepository } from "./DocumentationRepository.ts"; - -export class KitModuleDocumentationGenerator { - constructor( - private readonly collie: CollieRepository, - private readonly kitModules: KitModuleRepository, - private readonly controls: ComplianceControlRepository, - private readonly logger: Logger, - ) {} - - async generate(docsRepo: DocumentationRepository) { - const progress = new ProgressReporter( - "generating", - "kit module documentation", - this.logger, - ); - - // generate all kit module READMEs - const tasks = this.kitModules.all.map(async (x) => { - const dest = docsRepo.resolveKitModulePath(x.id); - - this.logger.verbose((fmt) => `generating ${fmt.kitPath(dest)}`); - - const md = this.generateModuleDocumentation(x, docsRepo); - - await Deno.mkdir(path.dirname(dest), { recursive: true }); - await Deno.writeTextFile(dest, md); - }); - - tasks.push(this.copyTopLevelKitReamde(docsRepo)); - - await Promise.all(tasks); - - progress.done(); - } - - private async copyTopLevelKitReamde(docsRepo: DocumentationRepository) { - const source = this.collie.resolvePath("kit", "README.md"); - const dest = docsRepo.resolveKitModulePath("README"); - await fs.ensureDir(path.dirname(dest)); - await fs.copy(source, dest, { overwrite: true }); - } - - private generateModuleDocumentation( - parsed: ParsedKitModule, - docsRepo: DocumentationRepository, - ) { - const complianceStatements = this.generateComplianceStatements( - parsed, - docsRepo, - ); - - if (!complianceStatements?.length) { - return parsed.readme; // return verbatim - } - - return `${parsed.readme} - -## Compliance Statements - -${complianceStatements.filter((x) => !!x).join("\n")} -`; - } - - private generateComplianceStatements( - parsed: ParsedKitModule, - docsRepo: DocumentationRepository, - ) { - return parsed.kitModule.compliance?.map((x) => { - const control = this.controls.tryFindById(x.control); - if (!control) { - this.logger.warn( - `could not find compliance control ${x.control} referenced in a compliance statement in ${parsed.definitionPath}`, - ); - - return; - } - - return ` -### ${control.name} - -${x.statement} - -[${control.name}](${ - docsRepo.controlLink( - docsRepo.resolveKitModulePath(parsed.id), - x.control, - ) - }) -`; - }); - } -} diff --git a/src/docs/PlatformDocumentationGenerator.ts b/src/docs/PlatformDocumentationGenerator.ts deleted file mode 100644 index 4c49c0d..0000000 --- a/src/docs/PlatformDocumentationGenerator.ts +++ /dev/null @@ -1,202 +0,0 @@ -import * as fs from "std/fs"; -import * as path from "std/path"; -import { TerragruntCliFacade } from "../api/terragrunt/TerragruntCliFacade.ts"; -import { Logger } from "../cli/Logger.ts"; -import { ProgressReporter } from "../cli/ProgressReporter.ts"; -import { - KitDependencyAnalyzer, - KitModuleDependency, - PlatformDependencies, -} from "../kit/KitDependencyAnalyzer.ts"; -import { CollieRepository } from "../model/CollieRepository.ts"; - -import { FoundationRepository } from "../model/FoundationRepository.ts"; -import { PlatformConfig } from "../model/PlatformConfig.ts"; -import { DocumentationRepository } from "./DocumentationRepository.ts"; -import { MarkdownUtils } from "../model/MarkdownUtils.ts"; -import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts"; -import { - RunIndividualPlatformModuleOutputCollector, -} from "./PlatformModuleOutputCollector.ts"; - -export class PlatformDocumentationGenerator { - constructor( - private readonly repo: CollieRepository, - private readonly foundation: FoundationRepository, - private readonly kitDependencyAnalyzer: KitDependencyAnalyzer, - private readonly controls: ComplianceControlRepository, - private readonly terragrunt: TerragruntCliFacade, - private readonly logger: Logger, - ) {} - - async generate(docsRepo: DocumentationRepository) { - await this.copyTopLevelPlatformsReamde(docsRepo); - await this.generatePlatformsDocumentation(docsRepo); - } - - private async copyTopLevelPlatformsReamde(docsRepo: DocumentationRepository) { - const source = this.foundation.resolvePath("platforms", "README.md"); - const dest = docsRepo.resolvePlatformsPath("README.md"); - - this.logger.verbose( - (fmt) => `Copying ${fmt.kitPath(source)} to ${fmt.kitPath(dest)}`, - ); - await fs.ensureDir(path.dirname(dest)); - await fs.copy(source, dest, { overwrite: true }); - } - - private async generatePlatformsDocumentation( - docsRepo: DocumentationRepository, - ) { - const foundationProgress = new ProgressReporter( - "generate documentation", - this.repo.relativePath(this.foundation.resolvePath()), - this.logger, - ); - - const foundationDependencies = await this.kitDependencyAnalyzer - .findKitModuleDependencies( - this.foundation, - ); - - for (const p of foundationDependencies.platforms) { - await this.generatePlatforDocumentation(p, docsRepo); - } - - foundationProgress.done(); - } - - private async generatePlatforDocumentation( - dependencies: PlatformDependencies, - docsRepo: DocumentationRepository, - ) { - const platformPath = this.foundation.resolvePlatformPath( - dependencies.platform, - ); - const platformProgress = new ProgressReporter( - "generate documentation", - this.repo.relativePath(platformPath), - this.logger, - ); - - const platformModuleDocumentation = - new RunIndividualPlatformModuleOutputCollector( - this.repo, - this.terragrunt, - this.logger, - ); - - // as a fallback process modules serially, unfortunately this is the only "safe" way to collect output - // see https://github.com/meshcloud/collie-cli/issues/265 - for (const dep of dependencies.modules) { - const documentationMd = await platformModuleDocumentation.getOutput(dep); - - await this.generatePlatformModuleDocumentation( - dep, - documentationMd, - docsRepo, - dependencies.platform, - ); - } - - platformProgress.done(); - } - - private async generatePlatformModuleDocumentation( - dep: KitModuleDependency, - documentationMd: string, - docsRepo: DocumentationRepository, - platform: PlatformConfig, - ) { - const destPath = docsRepo.resolvePlatformModulePath( - platform.id, - dep.kitModuleId, - ); - - await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes? - - const mdSections = [documentationMd]; - - const complianceStatementsBlock = this.generateComplianceStatementSection( - dep, - docsRepo, - destPath, - ); - mdSections.push(complianceStatementsBlock); - - const kitModuleSection = this.generateKitModuleSection( - dep, - docsRepo, - destPath, - ); - mdSections.push(kitModuleSection); - - await Deno.writeTextFile(destPath, mdSections.join("\n\n")); - - this.logger.verbose( - (fmt) => - `Wrote output "documentation_md" from platform module ${ - fmt.kitPath( - dep.sourcePath, - ) - } to ${fmt.kitPath(destPath)}`, - ); - } - - private generateKitModuleSection( - dep: KitModuleDependency, - docsRepo: DocumentationRepository, - destPath: string, - ) { - if (!dep.kitModule) { - return MarkdownUtils.container( - "warning", - "Invalid Kit Module Dependency", - "Could not find kit module at " + MarkdownUtils.code(dep.kitModulePath), - ); - } - - const kitModuleLink = MarkdownUtils.link( - dep.kitModule.name + " kit module", - docsRepo.kitModuleLink(destPath, dep.kitModuleId), - ); - - const kitModuleSection = `::: tip Kit module -This platform module is a deployment of kit module ${kitModuleLink}. -:::`; - return kitModuleSection; - } - - private generateComplianceStatementSection( - dep: KitModuleDependency, - docsRepo: DocumentationRepository, - destPath: string, - ) { - const complianceStatements = dep?.kitModule?.compliance - ?.map((x) => { - const control = this.controls.tryFindById(x.control); - if (!control) { - this.logger.warn( - `could not find compliance control ${x.control} referenced in a compliance statement in ${dep.kitModulePath}`, - ); - - return; - } - - return `- [${control.name}](${ - docsRepo.controlLink( - destPath, - x.control, - ) - }): ${x.statement}`; - }) - .filter((x): x is string => !!x); - - const complianceStatementsBlock = !complianceStatements?.length - ? `` - : `## Compliance Statements - -${complianceStatements.join("\n")}`; - return complianceStatementsBlock; - } -} diff --git a/src/docs/PlatformModuleOutputCollector.ts b/src/docs/PlatformModuleOutputCollector.ts deleted file mode 100644 index c46b41a..0000000 --- a/src/docs/PlatformModuleOutputCollector.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as path from "std/path"; -import { TerragruntCliFacade } from "../api/terragrunt/TerragruntCliFacade.ts"; -import { KitModuleDependency } from "../kit/KitDependencyAnalyzer.ts"; -import { CollieRepository } from "../model/CollieRepository.ts"; -import { Logger } from "../cli/Logger.ts"; -import { MeshError } from "../errors.ts"; - -/** - * Note: - * For a great UX/DX it's important that running "collie foundation docs" is fast. - * - * We have therefore tried speeding it up by collecting output from platform modules in parallel. - * Unfortunately, it appears that terragrunt does not offer us a good way to reliably get all the outputs from all - * platform modules, see https://github.com/meshcloud/collie-cli/issues/267 - * - * This "fast mode" detection also caused other bugs like https://github.com/meshcloud/collie-cli/issues/269 - * - * In the future, we should maybe investigate cachingas an alternative to parallelization, because usually an engineer - * would re-run "collie foundation docs" only after changing a specific platform module - */ - -export interface PlatformModuleOutputCollector { - getOutput(dep: KitModuleDependency): Promise; -} - -/** - * Collects platform module output by running each platform module individually in series. - */ -export class RunIndividualPlatformModuleOutputCollector - implements PlatformModuleOutputCollector { - constructor( - private readonly repo: CollieRepository, - private readonly terragrunt: TerragruntCliFacade, - private readonly logger: Logger, - ) {} - - async getOutput(dep: KitModuleDependency): Promise { - const result = await this.terragrunt.collectOutput( - this.repo.resolvePath(path.dirname(dep.sourcePath)), - "documentation_md", - ); - - if (!result.status.success) { - this.logger.error( - (fmt) => - `Failed to collect output "documentation_md" from platform module ${ - fmt.kitPath( - dep.sourcePath, - ) - }`, - ); - this.logger.error(result.stderr); - - throw new MeshError( - "Failed to collect documentation output from platform modules", - ); - } - - return result.stdout; - } -} diff --git a/src/kit/KitDependencyAnalyzer.ts b/src/kit/KitDependencyAnalyzer.ts index e54e26c..4eb07a6 100644 --- a/src/kit/KitDependencyAnalyzer.ts +++ b/src/kit/KitDependencyAnalyzer.ts @@ -5,6 +5,7 @@ import { KitModule } from "./KitModule.ts"; import { Logger } from "../cli/Logger.ts"; import { ProgressReporter } from "../cli/ProgressReporter.ts"; import { CollieRepository } from "../model/CollieRepository.ts"; +import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts"; export interface FoundationDependencies { foundation: string; @@ -42,6 +43,7 @@ export class KitDependencyAnalyzer { constructor( private readonly collie: CollieRepository, private readonly kitModules: KitModuleRepository, + private readonly complianceControls: ComplianceControlRepository, private readonly logger: Logger, ) {} @@ -112,7 +114,10 @@ export class KitDependencyAnalyzer { const msg = `Could not find kit module with id ${kitModuleId} included from ${sourcePath}`; this.logger.warn(msg); + } else { + this.validateKitModuleComplianceStatements(kitModule, kitModulePath); } + return { sourcePath, kitModuleId, @@ -120,6 +125,18 @@ export class KitDependencyAnalyzer { kitModule, }; } + validateKitModuleComplianceStatements( + kitModule: KitModule, + kitModulePath: string, + ) { + (kitModule?.compliance || []).forEach((x) => { + if (!this.complianceControls.tryFindById(x.control)) { + this.logger.warn( + `Could not find compliance control ${x.control} referenced in a compliance statement in ${kitModulePath}`, + ); + } + }); + } static parseTerraformSource(hcl: string): string | undefined { const regex = /terraform {[\s\S]+source = "([\s\S]+?)"/g;