From 73e542dbd33292dd7327ad0c37798c357109168e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Sep 2023 11:52:28 +0200 Subject: [PATCH] Fixes #23469: Reporting by node is not correct on directives and Rules --- .../rudder/domain/reports/StatusReports.scala | 22 +- .../rudder/rest/data/Compliance.scala | 258 ++++++++++++++++-- .../rudder/rest/lift/ComplianceApi.scala | 86 +++--- .../rest/TestDirectiveComplianceCsv.scala | 11 +- 4 files changed, 281 insertions(+), 96 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala index 738754a7ed8..7fc33c4476d 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala @@ -328,7 +328,8 @@ final case class BlockStatusReport( subComponents.find(_.componentName == componentName).toList ::: subComponents.collect { case g: BlockStatusReport => g }.flatMap(_.findChildren(componentName)) } - def compliance: ComplianceLevel = { + + override lazy val compliance: ComplianceLevel = { import ReportingLogic._ reportingLogic match { // simple weighted compliance, as usual @@ -343,7 +344,8 @@ final case class BlockStatusReport( } ComplianceLevel.compute(kept) // focus on a given sub-component name (can be present several time, or 0 which leads to N/A) - case FocusReport(component) => ComplianceLevel.sum(findChildren(component).map(_.compliance)) + case FocusReport(component) => + ComplianceLevel.sum(findChildren(component).map(_.compliance)) } } @@ -351,15 +353,16 @@ final case class BlockStatusReport( subComponents.flatMap(_.getValues(predicate)) } - def componentValues: List[ComponentValueStatusReport] = ComponentValueStatusReport.merge(getValues(_ => true)) - def withFilteredElement(predicate: ComponentValueStatusReport => Boolean): Option[ComponentStatusReport] = { + val componentValues: List[ComponentValueStatusReport] = ComponentValueStatusReport.merge(getValues(_ => true)) + + def withFilteredElement(predicate: ComponentValueStatusReport => Boolean): Option[ComponentStatusReport] = { subComponents.flatMap(_.withFilteredElement(predicate)) match { case Nil => None case l => Some(this.copy(subComponents = l)) } } - def status: ReportType = { + val status: ReportType = { reportingLogic match { case WorstReportWeightedOne | WorstReportWeightedSum | WeightedReport => ReportType.getWorseType(subComponents.map(_.status)) @@ -370,14 +373,14 @@ final case class BlockStatusReport( } final case class ValueStatusReport( componentName: String, - expectedComponentName: String, // only one ComponentValueStatusReport by valuex. - - componentValues: List[ComponentValueStatusReport] + expectedComponentName: String, // only one ComponentValueStatusReport by values. + componentValues: List[ComponentValueStatusReport] ) extends ComponentStatusReport { override def toString() = s"${componentName}:${componentValues.toSeq.sortBy(_.componentValue).mkString("[", ",", "]")}" override lazy val compliance = ComplianceLevel.sum(componentValues.map(_.compliance)) + /* * Get all values matching the predicate */ @@ -385,7 +388,8 @@ final case class ValueStatusReport( componentValues.filter(v => predicate(v)).toSeq } - def status: ReportType = ReportType.getWorseType(getValues(_ => true).map(_.status)) + val status: ReportType = ReportType.getWorseType(getValues(_ => true).map(_.status)) + /* * Rebuild a componentStatusReport, keeping only values matching the predicate */ diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala index 3e617a88459..80421e85296 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala @@ -37,6 +37,12 @@ package com.normation.rudder.rest.data +import com.normation.cfclerk.domain.ReportingLogic +import com.normation.cfclerk.domain.ReportingLogic.FocusReport +import com.normation.cfclerk.domain.ReportingLogic.WeightedReport +import com.normation.cfclerk.domain.ReportingLogic.WorstReportWeightedOne +import com.normation.cfclerk.domain.ReportingLogic.WorstReportWeightedSum +import com.normation.cfclerk.domain.WorstReportReportingLogic import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.PolicyMode @@ -131,27 +137,143 @@ final case class ByRuleDirectiveCompliance( sealed trait ByRuleComponentCompliance { def name: String def compliance: ComplianceLevel + + def getValues(predicate: ByRuleNodeCompliance => Boolean): Seq[ByRuleNodeCompliance] + + def withFilteredElement(predicate: ByRuleNodeCompliance => Boolean): Option[ByRuleComponentCompliance] + + def componentValues: List[ByRuleNodeCompliance] + + def componentValues(v: String): List[ByRuleNodeCompliance] = componentValues.filter(_.values.exists(_.componentValue == v)) + + def status: ReportType +} + +object ByRuleComponentCompliance { + + def merge(components: Iterable[ByRuleComponentCompliance]): List[ByRuleComponentCompliance] = { + components.groupBy(_.name).flatMap { + case (cptName, reports) => + val valueComponents = reports.collect { case c: ByRuleValueCompliance => c }.toList + ByRuleValueCompliance(cptName, valueComponents.flatMap(_.nodes)) + + val groupComponent = reports.collect { case c: ByRuleBlockCompliance => c }.toList match { + case Nil => Nil + case r => + import ReportingLogic._ + val reportingLogic = r + .map(_.reportingLogic) + .reduce((a, b) => { + (a, b) match { + case (WorstReportWeightedOne, _) | (_, WorstReportWeightedOne) => WorstReportWeightedOne + case (WorstReportWeightedSum, _) | (_, WorstReportWeightedSum) => WorstReportWeightedSum + case (WeightedReport, _) | (_, WeightedReport) => WeightedReport + case (FocusReport(a), _) => FocusReport(a) + } + }) + ByRuleBlockCompliance(cptName, ByRuleComponentCompliance.merge(r.flatMap(_.subComponents)), reportingLogic) :: Nil + } + groupComponent ::: valueComponents + + } + }.toList } final case class ByRuleBlockCompliance( - name: String, - compliance: ComplianceLevel, - subComponents: Seq[ByRuleComponentCompliance] -) extends ByRuleComponentCompliance + name: String, + subComponents: Seq[ByRuleComponentCompliance], + reportingLogic: ReportingLogic +) extends ByRuleComponentCompliance { + def findChildren(componentName: String): List[ByRuleComponentCompliance] = { + subComponents.find(_.name == componentName).toList ::: + subComponents.collect { case g: ByRuleBlockCompliance => g }.flatMap(_.findChildren(componentName)).toList + } + + override lazy val compliance: ComplianceLevel = { + import ReportingLogic._ + reportingLogic match { + // simple weighted compliance, as usual + case WeightedReport => ComplianceLevel.sum(subComponents.map(_.compliance)) + // worst case bubble up, and its weight can be either 1 or the sum of sub-component weight + case worst: WorstReportReportingLogic => + val worstReport = ReportType.getWorseType(subComponents.map(_.status)) + val allReports = getValues(_ => true).flatMap(_.values.flatMap(_.messages.map(_ => worstReport))) + val kept = worst match { + case WorstReportWeightedOne => allReports.take(1) + case WorstReportWeightedSum => allReports + } + ComplianceLevel.compute(kept) + // focus on a given sub-component name (can be present several time, or 0 which leads to N/A) + case FocusReport(component) => + ComplianceLevel.sum(findChildren(component).map(_.compliance)) + } + } + + def getValues(predicate: ByRuleNodeCompliance => Boolean): Seq[ByRuleNodeCompliance] = { + subComponents.flatMap(_.getValues(predicate)) + } + + val componentValues: List[ByRuleNodeCompliance] = getValues(_ => true).toList + + def withFilteredElement(predicate: ByRuleNodeCompliance => Boolean): Option[ByRuleComponentCompliance] = { + subComponents.flatMap(_.withFilteredElement(predicate)) match { + case Nil => None + case l => Some(this.copy(subComponents = l)) + } + } + + val status: ReportType = { + reportingLogic match { + case WorstReportWeightedOne | WorstReportWeightedSum | WeightedReport => + ReportType.getWorseType(subComponents.map(_.status)) + case FocusReport(component) => + ReportType.getWorseType(findChildren(component).map(_.status)) + } + } + +} final case class ByRuleValueCompliance( - name: String, - compliance: ComplianceLevel, - nodes: Seq[ByRuleNodeCompliance] -) extends ByRuleComponentCompliance + name: String, + // compliance: ComplianceLevel, + nodes: Seq[ByRuleNodeCompliance] +) extends ByRuleComponentCompliance { + + override lazy val compliance = ComplianceLevel.sum(componentValues.map(_.compliance)) + + /* + * Get all values matching the predicate + */ + def getValues(predicate: ByRuleNodeCompliance => Boolean): Seq[ByRuleNodeCompliance] = { + nodes.filter(v => predicate(v)).toSeq + } + + val status: ReportType = ReportType.getWorseType(getValues(_ => true).map(_.status)) + + val componentValues = nodes.toList + + /* + * Rebuild a componentStatusReport, keeping only values matching the predicate + */ + def withFilteredElement(predicate: ByRuleNodeCompliance => Boolean): Option[ByRuleComponentCompliance] = { + val values = componentValues.filter { case v => predicate(v) } + if (values.isEmpty) None + else Some(this.copy(nodes = values)) + } + +} final case class ByRuleNodeCompliance( - id: NodeId, - name: String, - mode: Option[PolicyMode], - compliance: ComplianceLevel, - values: Seq[ComponentValueStatusReport] -) + id: NodeId, + name: String, + mode: Option[PolicyMode], + // compliance: ComplianceLevel, + values: Seq[ComponentValueStatusReport] +) { + lazy val compliance = ComplianceLevel.sum(values.map(_.compliance)) + + val status: ReportType = ReportType.getWorseType(values.map(_.status)) +} /* This is the same compliance structure than ByRuleByDirectiveCompliance except that the entry point is a Node The full hierarchy is @@ -179,19 +301,101 @@ final case class ByRuleByNodeByDirectiveCompliance( sealed trait ByRuleByNodeByDirectiveByComponentCompliance { def name: String def compliance: ComplianceLevel + + def getValues(predicate: ComponentValueStatusReport => Boolean): Seq[ComponentValueStatusReport] + + def withFilteredElement(predicate: ComponentValueStatusReport => Boolean): Option[ByRuleByNodeByDirectiveByComponentCompliance] + + def componentValues: List[ComponentValueStatusReport] + + def componentValues(v: String): List[ComponentValueStatusReport] = componentValues.filter(_.componentValue == v) + + def status: ReportType } final case class ByRuleByNodeByDirectiveByBlockCompliance( - name: String, - compliance: ComplianceLevel, - subComponents: Seq[ByRuleByNodeByDirectiveByComponentCompliance] -) extends ByRuleByNodeByDirectiveByComponentCompliance + name: String, + subComponents: Seq[ByRuleByNodeByDirectiveByComponentCompliance], + reportingLogic: ReportingLogic +) extends ByRuleByNodeByDirectiveByComponentCompliance { + + def findChildren(componentName: String): List[ByRuleByNodeByDirectiveByComponentCompliance] = { + subComponents.find(_.name == componentName).toList ::: + subComponents.collect { case g: ByRuleByNodeByDirectiveByBlockCompliance => g }.flatMap(_.findChildren(componentName)).toList + } + + override lazy val compliance: ComplianceLevel = { + import ReportingLogic._ + reportingLogic match { + // simple weighted compliance, as usual + case WeightedReport => ComplianceLevel.sum(subComponents.map(_.compliance)) + // worst case bubble up, and its weight can be either 1 or the sum of sub-component weight + case worst: WorstReportReportingLogic => + val worstReport = ReportType.getWorseType(subComponents.map(_.status)) + val allReports = getValues(_ => true).flatMap(_.messages.map(_ => worstReport)) + val kept = worst match { + case WorstReportWeightedOne => allReports.take(1) + case WorstReportWeightedSum => allReports + } + ComplianceLevel.compute(kept) + // focus on a given sub-component name (can be present several time, or 0 which leads to N/A) + case FocusReport(component) => + ComplianceLevel.sum(findChildren(component).map(_.compliance)) + } + } + + def getValues(predicate: ComponentValueStatusReport => Boolean): Seq[ComponentValueStatusReport] = { + subComponents.flatMap(_.getValues(predicate)) + } + + val componentValues: List[ComponentValueStatusReport] = ComponentValueStatusReport.merge(getValues(_ => true)) + + def withFilteredElement( + predicate: ComponentValueStatusReport => Boolean + ): Option[ByRuleByNodeByDirectiveByComponentCompliance] = { + subComponents.flatMap(_.withFilteredElement(predicate)) match { + case Nil => None + case l => Some(this.copy(subComponents = l)) + } + } + + val status: ReportType = { + reportingLogic match { + case WorstReportWeightedOne | WorstReportWeightedSum | WeightedReport => + ReportType.getWorseType(subComponents.map(_.status)) + case FocusReport(component) => + ReportType.getWorseType(findChildren(component).map(_.status)) + } + } + +} final case class ByRuleByNodeByDirectiveByValueCompliance( - name: String, - compliance: ComplianceLevel, - values: Seq[ComponentValueStatusReport] -) extends ByRuleByNodeByDirectiveByComponentCompliance + name: String, + componentValues: List[ComponentValueStatusReport] +) extends ByRuleByNodeByDirectiveByComponentCompliance { + override lazy val compliance = ComplianceLevel.sum(componentValues.map(_.compliance)) + + /* + * Get all values matching the predicate + */ + def getValues(predicate: ComponentValueStatusReport => Boolean): Seq[ComponentValueStatusReport] = { + componentValues.filter(v => predicate(v)).toSeq + } + + val status: ReportType = ReportType.getWorseType(getValues(_ => true).map(_.status)) + + /* + * Rebuild a componentStatusReport, keeping only values matching the predicate + */ + def withFilteredElement( + predicate: ComponentValueStatusReport => Boolean + ): Option[ByRuleByNodeByDirectiveByComponentCompliance] = { + val values = componentValues.filter { case v => predicate(v) } + if (values.isEmpty) None + else Some(this.copy(componentValues = values)) + } +} object GroupComponentCompliance { // This function do the recursive treatment of components, we will have each time a pair of Sequence of tuple (NodeId , component compliance structure) @@ -212,7 +416,8 @@ object GroupComponentCompliance { // All subComponents are regrouped by Node, rebuild our block for each node case (nodeId, s) => val subs = s.map(_._2) - (nodeId, ByRuleByNodeByDirectiveByBlockCompliance(b.name, ComplianceLevel.sum(subs.map(_.compliance)), subs)) + + (nodeId, ByRuleByNodeByDirectiveByBlockCompliance(b.name, subs, b.reportingLogic)) } .toSeq // Value case @@ -227,8 +432,7 @@ object GroupComponentCompliance { (nodeId, data.map(_.name).headOption.getOrElse(nodeId.value), data.map(_.mode).headOption.getOrElse(None)), ByRuleByNodeByDirectiveByValueCompliance( v.name, - ComplianceLevel.sum(data.map(_.compliance)), - data.flatMap(_.values) + data.flatMap(_.values).toList ) ) }).toSeq @@ -492,7 +696,7 @@ object JsonCompliance { case component: ByRuleByNodeByDirectiveByBlockCompliance => ("components" -> byNodeByDirectiveByComponents(component.subComponents, level, precision)) case component: ByRuleByNodeByDirectiveByValueCompliance => - ("values" -> values(component.values, level)) + ("values" -> values(component.componentValues, level)) }) ) }) @@ -724,7 +928,7 @@ object JsonCompliance { case component: ByRuleByNodeByDirectiveByBlockCompliance => ("components" -> byNodeByDirectiveByComponents(component.subComponents, level, precision)) case component: ByRuleByNodeByDirectiveByValueCompliance => - ("values" -> values(component.values, level)) + ("values" -> values(component.componentValues, level)) }) ) }) diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala index 4bf94b16d82..5647b5ec86f 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala @@ -38,6 +38,7 @@ package com.normation.rudder.rest.lift import com.normation.box._ +import com.normation.cfclerk.domain.ReportingLogic import com.normation.errors._ import com.normation.inventory.domain.NodeId import com.normation.rudder.api.ApiVersion @@ -389,47 +390,27 @@ class ComplianceAPIService( private[this] def components( nodeInfos: Map[NodeId, NodeInfo] - )(name: String, nodeComponents: List[(NodeId, ComponentStatusReport)]): List[ByRuleComponentCompliance] = { + )(component: ComponentStatusReport, nodeId: NodeId): ByRuleComponentCompliance = { + component match { + case BlockStatusReport(componentName, reportingLogic, subComponents) => + ByRuleBlockCompliance( + componentName, + subComponents.map(sub => components(nodeInfos)(sub, nodeId)), + reportingLogic + ) - val (groupsComponents, uniqueComponents) = nodeComponents.partitionMap { - case (a, b: BlockStatusReport) => Left((a, b)) - case (a, b: ValueStatusReport) => Right((a, b)) + case ValueStatusReport(componentName, expectedComponentName, componentValues) => + val optNodeInfo = nodeInfos.get(nodeId) + ByRuleValueCompliance( + componentName, + ByRuleNodeCompliance( + nodeId, + optNodeInfo.map(_.hostname).getOrElse("Unknown node"), + optNodeInfo.map(_.policyMode).getOrElse(None), + componentValues + ) :: Nil + ) } - - (if (groupsComponents.isEmpty) { - Nil - } else { - val bidule = groupsComponents.flatMap { case (nodeId, c) => c.subComponents.map(sub => (nodeId, sub)) } - .groupBy(_._2.componentName) - ByRuleBlockCompliance( - name, - ComplianceLevel.sum(groupsComponents.map(_._2.compliance)), - bidule.flatMap(c => components(nodeInfos)(c._1, c._2)).toList - ) :: Nil - }) ::: (if (uniqueComponents.isEmpty) { - Nil - } else { - ByRuleValueCompliance( - name, - ComplianceLevel.sum( - uniqueComponents.map(_._2.compliance) - ), // here, we finally group by nodes for each components ! - { - val byNode = uniqueComponents.groupBy(_._1) - byNode.map { - case (nodeId, components) => - val optNodeInfo = nodeInfos.get(nodeId) - ByRuleNodeCompliance( - nodeId, - optNodeInfo.map(_.hostname).getOrElse("Unknown node"), - optNodeInfo.map(_.policyMode).getOrElse(None), - ComplianceLevel.sum(components.map(_._2.compliance)), - components.sortBy(_._2.componentName).flatMap(_._2.componentValues) - ) - }.toSeq - } - ) :: Nil - }) } private[this] def getByDirectivesCompliance( @@ -480,20 +461,20 @@ class ComplianceAPIService( (ruleId, ruleReports) <- reportsByRule.toSeq // We will now gather our report by component for the current Directive - reportsByComponents = (for { + reportsByComponents = for { ruleReport <- ruleReports nodeId = ruleReport.nodeId directiveReports <- ruleReport.directives.get(directive.id).toList component <- directiveReports.components } yield { - (nodeId, component) - }).groupBy(_._2.componentName).toSeq + components(nodeInfos)(component, nodeId) + } } yield { - val componentsCompliance = reportsByComponents.flatMap(c => components(nodeInfos)(c._1, c._2.toList)) - val ruleName = rules.find(_.id == ruleId).map(_.name).getOrElse("") - val componentsDetails = if (computedLevel <= 3) Seq() else componentsCompliance + val componentsCompliance = ByRuleComponentCompliance.merge(reportsByComponents) + val ruleName = rules.find(_.id == ruleId).map(_.name).getOrElse("") + val componentsDetails = if (computedLevel <= 3) Seq() else componentsCompliance ByDirectiveByRuleCompliance( ruleId, @@ -591,15 +572,16 @@ class ComplianceAPIService( ), // here we want the compliance by components of the directive. // if level is high enough, get all components and group by their name { - val byComponents: Map[String, immutable.Iterable[(NodeId, ComponentStatusReport)]] = if (computedLevel < 3) { - Map() + val byComponents = if (computedLevel < 3) { + Nil } else { - nodeDirectives.flatMap { case (nodeId, d) => d.components.map(c => (nodeId, c)).toSeq } - .groupBy(_._2.componentName) + + nodeDirectives.flatMap { + case (nodeId, d) => d.components.map(c => components(nodeInfos)(c, nodeId)) + }.toList + } - byComponents.flatMap { - case (name, nodeComponents) => components(nodeInfos)(name, nodeComponents.toList) - }.toSeq + ByRuleComponentCompliance.merge(byComponents) } ) }.toSeq diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/TestDirectiveComplianceCsv.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/TestDirectiveComplianceCsv.scala index 10b9a310a36..6ee48570e5c 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/TestDirectiveComplianceCsv.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/TestDirectiveComplianceCsv.scala @@ -37,6 +37,7 @@ package com.normation.rudder.rest +import com.normation.cfclerk.domain.ReportingLogic.WeightedReport import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.DirectiveUid @@ -104,17 +105,14 @@ class TestDirectiveComplianceCsv extends Specification { Seq( ByRuleBlockCompliance( "Check Cipher TLS_RSA_WITH_DES_CBC_SHA", - notUsed, Seq( ByRuleValueCompliance( "Command execution", - notUsed, Seq( ByRuleNodeCompliance( NodeId("n1"), "prod-www-01.lab.rudder.io", None, - notUsed, Seq( ComponentValueStatusReport( "Disable-TlsCipherSuite -Name \"TLS_RSA_WITH_DES_CBC_SHA\" ", @@ -134,7 +132,6 @@ class TestDirectiveComplianceCsv extends Specification { NodeId("n1"), "prod-windows-2016.demo.normation.com", None, - notUsed, Seq( ComponentValueStatusReport( "Disable-TlsCipherSuite -Name \"TLS_RSA_WITH_DES_CBC_SHA\" ", @@ -154,13 +151,11 @@ class TestDirectiveComplianceCsv extends Specification { ), ByRuleValueCompliance( "Audit from Powershell execution", - notUsed, Seq( ByRuleNodeCompliance( NodeId("n1"), "prod-app-01.lab.rudder.io", None, - notUsed, Seq( ComponentValueStatusReport( "(Get-TlsCipherSuite -Name \"TLS_RSA_WITH_DES_CBC_SHA\").Count", @@ -180,7 +175,6 @@ class TestDirectiveComplianceCsv extends Specification { NodeId("n1"), "prod-windows-2016.demo.normation.com", None, - notUsed, Seq( ComponentValueStatusReport( "(Get-TlsCipherSuite -Name \"TLS_RSA_WITH_DES_CBC_SHA\").Count", @@ -198,7 +192,8 @@ class TestDirectiveComplianceCsv extends Specification { ) ) ) - ) + ), + WeightedReport ) ) )