From ddcd3978b174ab9e96c700581de7ec68023e017b Mon Sep 17 00:00:00 2001 From: "david.watkins@db.com" Date: Tue, 7 May 2024 15:11:43 +0100 Subject: [PATCH 1/8] Flow Classification re-eval on rule change #CTCTOWALTZ-3177 --- .../finos/waltz/common/FunctionUtilities.java | 3 +- .../FlowClassificationRuleDao.java | 23 +++++++ .../FlowClassificationRuleHarness.java | 63 ++++++++++++++++-- .../FlowClassificationRuleService.java | 65 ++++++++++--------- 4 files changed, 115 insertions(+), 39 deletions(-) diff --git a/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java index 72df903b0e..0b1b79e6ad 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java @@ -66,10 +66,11 @@ public static T time(String name, Supplier supplier) { long st = System.currentTimeMillis(); try { + LOG.info("----> begin [{}]", name); T r = supplier.get(); long end = System.currentTimeMillis(); - LOG.info("duration [{}]: {}", name, (end - st)); + LOG.info("<---- end [{}], duration:{}", name, (end - st)); return r; } catch (Exception e) { String msg = String.format("Unexpected error when timing [%s]: %s", name, e.getMessage()); diff --git a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java index 17f9eddd75..2d1f54a8c9 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java @@ -24,6 +24,7 @@ import org.finos.waltz.model.FlowDirection; import org.finos.waltz.model.ImmutableEntityReference; import org.finos.waltz.model.MessageSeverity; +import org.finos.waltz.model.datatype.FlowDataType; import org.finos.waltz.model.flow_classification_rule.DiscouragedSource; import org.finos.waltz.model.flow_classification_rule.FlowClassificationRule; import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleCreateCommand; @@ -414,6 +415,28 @@ public List findFlowClassificationRuleVantag } + public List findFlowClassificationRuleVantagePoints(FlowDirection direction, + Set dataTypeIdsToConsider) { + return findFlowClassificationRuleVantagePoints( + FLOW_CLASSIFICATION.DIRECTION.eq(direction.name()) + .and(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.in(dataTypeIdsToConsider))); + } + + + + public List findFlowClassificationRuleVantagePoints(FlowDirection direction, + org.finos.waltz.model.entity_hierarchy.EntityHierarchy dtHierarchy, + Set population) { + Set possibleDtIds = population + .stream() + .map(FlowDataType::dtId) + .distinct() + .flatMap(dtId -> dtHierarchy.findAncestors(dtId).stream()) + .collect(Collectors.toSet()); + return findFlowClassificationRuleVantagePoints(direction, possibleDtIds); + } + + private List findFlowClassificationRuleVantagePoints(Condition condition) { SelectSeekStep6, String, Integer, Integer, Long, Long, Long> select = dsl .select(vantagePointId, diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java index 5922f03041..3e586f1ee9 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java @@ -18,31 +18,80 @@ package org.finos.waltz.jobs.harness; +import org.finos.waltz.data.datatype_decorator.LogicalFlowDecoratorDao; +import org.finos.waltz.data.flow_classification_rule.FlowClassificationRuleDao; +import org.finos.waltz.data.logical_flow.LogicalFlowIdSelectorFactory; import org.finos.waltz.model.EntityKind; -import org.finos.waltz.model.flow_classification_rule.FlowClassificationRule; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.FlowDirection; +import org.finos.waltz.model.IdSelectionOptions; +import org.finos.waltz.model.datatype.FlowDataType; +import org.finos.waltz.model.entity_hierarchy.EntityHierarchy; +import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleVantagePoint; import org.finos.waltz.service.DIConfiguration; +import org.finos.waltz.service.entity_hierarchy.EntityHierarchyService; import org.finos.waltz.service.flow_classification_rule.FlowClassificationRuleService; import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Select; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; -import static org.finos.waltz.model.EntityReference.mkRef; -import static org.finos.waltz.model.IdSelectionOptions.mkOpts; +import static org.finos.waltz.common.FunctionUtilities.time; public class FlowClassificationRuleHarness { + private static final LogicalFlowIdSelectorFactory logicalFlowIdSelectorFactory = new LogicalFlowIdSelectorFactory(); + + public static void main(String[] args) { AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); DSLContext dsl = ctx.getBean(DSLContext.class); - FlowClassificationRuleService svc = ctx.getBean(FlowClassificationRuleService.class); - Set r = svc.findClassificationRules(mkOpts(mkRef(EntityKind.ORG_UNIT, 14902L))); - System.out.println(r); + FlowClassificationRuleService fcrSvc = ctx.getBean(FlowClassificationRuleService.class); + EntityHierarchyService ehSvc = ctx.getBean(EntityHierarchyService.class); + FlowClassificationRuleDao fcrDao = ctx.getBean(FlowClassificationRuleDao.class); + LogicalFlowDecoratorDao lfdDao = ctx.getBean(LogicalFlowDecoratorDao.class); + + EntityReference collibra = EntityReference.mkRef(EntityKind.APPLICATION, 124334001L); + EntityReference fdw = EntityReference.mkRef(EntityKind.APPLICATION, 16831L); + EntityReference rftEds = EntityReference.mkRef(EntityKind.ORG_UNIT, 10526L); + EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 95L); + + Select> flowSelector = mkSelector(group); + EntityHierarchy dtHierarchy = time("loading dt hier", () -> ehSvc.fetchHierarchyForKind(EntityKind.DATA_TYPE)); + Set population = time("lfd::fetchPopulation", () -> lfdDao.fetchFlowDataTypePopulationForFlowSelector(flowSelector)); + Set possibleDtIds = population + .stream() + .map(FlowDataType::dtId) + .distinct() + .flatMap(dtId -> dtHierarchy.findAncestors(dtId).stream()) + .collect(Collectors.toSet()); + + List allRules = time("fcrDao::findRules (all)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND)); + List targetedDtRules = time("fcrDao::findRules (targeted - dt)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, possibleDtIds)); + List targetedPopRules = time("fcrDao::findRules (targeted- pop)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, dtHierarchy, population)); + + System.out.printf( + "\nRules: pop:%d / dt:%d / all:%d, Population: %d, DTs: %d\n\n", + targetedPopRules.size(), + targetedDtRules.size(), + allRules.size(), + population.size(), + possibleDtIds.size()); + + time("recalc", () -> fcrSvc.recalculateRatingsForPopulation(population)); + } -// System.exit(-1); + private static Select> mkSelector(EntityReference ref) { + IdSelectionOptions opts = IdSelectionOptions.mkOpts(ref); + Select> flowSelector = logicalFlowIdSelectorFactory.apply(opts); + return flowSelector; } diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java index e6857fb003..b53983a0d4 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java @@ -233,11 +233,15 @@ public int fastRecalculateAllFlowRatings() { return recalculateRatingsForPopulation(population); } - private int recalculateRatingsForPopulation(Set population) { + public int recalculateRatingsForPopulation(Set population) { + + LOG.debug("Loading hierarchies"); + EntityHierarchy ouHierarchy = time("loading ou hier", () -> entityHierarchyService.fetchHierarchyForKind(ORG_UNIT)); + EntityHierarchy dtHierarchy = time("loading dt hier", () -> entityHierarchyService.fetchHierarchyForKind(EntityKind.DATA_TYPE)); LOG.debug("Loading rule vantage points"); - List inboundRuleVantagePoints = flowClassificationRuleDao.findFlowClassificationRuleVantagePoints(FlowDirection.INBOUND); - List outboundRuleVantagePoints = flowClassificationRuleDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND); + List inboundRuleVantagePoints = flowClassificationRuleDao.findFlowClassificationRuleVantagePoints(FlowDirection.INBOUND, dtHierarchy, population); + List outboundRuleVantagePoints = flowClassificationRuleDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, dtHierarchy, population); Map inboundRatingCodeByRuleId = indexBy(inboundRuleVantagePoints, FlowClassificationRuleVantagePoint::ruleId, FlowClassificationRuleVantagePoint::classificationCode); Map outboundRatingCodeByRuleId = indexBy(outboundRuleVantagePoints, FlowClassificationRuleVantagePoint::ruleId, FlowClassificationRuleVantagePoint::classificationCode); @@ -249,10 +253,6 @@ private int recalculateRatingsForPopulation(Set population) { outboundRuleVantagePoints.size(), population.size()); - LOG.debug("Loading hierarchies"); - EntityHierarchy ouHierarchy = entityHierarchyService.fetchHierarchyForKind(ORG_UNIT); - EntityHierarchy dtHierarchy = entityHierarchyService.fetchHierarchyForKind(EntityKind.DATA_TYPE); - LOG.debug("Applying rules to population"); Map> lfdIdToOutboundRuleIdMap = time("outbound vps", () -> applyVantagePoints( FlowDirection.OUTBOUND, @@ -267,29 +267,32 @@ private int recalculateRatingsForPopulation(Set population) { ouHierarchy, dtHierarchy)); - LOG.debug("Calculating diff"); - Set> existingDecoratorRatingInfo = map( - population, - d -> tuple(d.lfdId(), d.sourceOutboundRating(), d.targetInboundRating(), d.outboundRuleId(), d.inboundRuleId())); - - Set> requiredDecoratorRatingInfo = mkRequiredDecoratorRatingInfo( - population, - outboundRatingCodeByRuleId, - inboundRatingCodeByRuleId, - lfdIdToOutboundRuleIdMap, - lfdIdToInboundRuleIdMap); - - - DiffResult> decoratorRatingDiff = mkDiff( - existingDecoratorRatingInfo, - requiredDecoratorRatingInfo, - d -> d.v1, - (newRecord, existingRecord) -> { - boolean sameOutboundFcr = (newRecord.v4 == null && existingRecord.v4 == null) || Objects.equals(newRecord.v4, existingRecord.v4); - boolean sameInboundFcr = (newRecord.v5 == null && existingRecord.v5 == null) || Objects.equals(newRecord.v5, existingRecord.v5); - boolean sameOutboundRating = newRecord.v2.value().equals(existingRecord.v2.value()); - boolean sameInboundRating = newRecord.v3.value().equals(existingRecord.v3.value()); - return sameOutboundRating && sameInboundRating && sameOutboundFcr && sameInboundFcr; + DiffResult> decoratorRatingDiff = time( + "calculating diff", + () -> { + Set> existingDecoratorRatingInfo = map( + population, + d -> tuple(d.lfdId(), d.sourceOutboundRating(), d.targetInboundRating(), d.outboundRuleId(), d.inboundRuleId())); + + Set> requiredDecoratorRatingInfo = mkRequiredDecoratorRatingInfo( + population, + outboundRatingCodeByRuleId, + inboundRatingCodeByRuleId, + lfdIdToOutboundRuleIdMap, + lfdIdToInboundRuleIdMap); + + + return mkDiff( + existingDecoratorRatingInfo, + requiredDecoratorRatingInfo, + d -> d.v1, + (newRecord, existingRecord) -> { + boolean sameOutboundFcr = (newRecord.v4 == null && existingRecord.v4 == null) || Objects.equals(newRecord.v4, existingRecord.v4); + boolean sameInboundFcr = (newRecord.v5 == null && existingRecord.v5 == null) || Objects.equals(newRecord.v5, existingRecord.v5); + boolean sameOutboundRating = newRecord.v2.value().equals(existingRecord.v2.value()); + boolean sameInboundRating = newRecord.v3.value().equals(existingRecord.v3.value()); + return sameOutboundRating && sameInboundRating && sameOutboundFcr && sameInboundFcr; + }); }); LOG.debug("Preparing to update {} logical flow decorators with new rating classifications", decoratorRatingDiff.differingIntersection().size()); @@ -303,7 +306,7 @@ private int recalculateRatingsForPopulation(Set population) { .set(LOGICAL_FLOW_DECORATOR.INBOUND_FLOW_CLASSIFICATION_RULE_ID, d.v5) .where(LOGICAL_FLOW_DECORATOR.ID.eq(d.v1))); - int updatedRecords = flowClassificationRuleDao.updateDecoratorsWithClassifications(updateStmts); + int updatedRecords = time("performing updates", () -> flowClassificationRuleDao.updateDecoratorsWithClassifications(updateStmts)); LOG.debug("Updated {} logical flow decorators with a classification", updatedRecords); return updatedRecords; From c0ae07883e257bd7ac0add5c44551b7c0879d21c Mon Sep 17 00:00:00 2001 From: woodjes Date: Wed, 8 May 2024 15:49:28 +0100 Subject: [PATCH 2/8] Improve performance of the fcr outcomes #CTCTOWALTZ-3177 #7067 --- .../org/finos/waltz/common/MapUtilities.java | 22 +++ .../FlowClassificationRuleHarness.java | 1 + .../data_type/DataTypeDecoratorService.java | 4 +- .../FlowClassificationRuleService.java | 11 +- .../FlowClassificationRuleUtilities.java | 161 +++++++++++++++--- 5 files changed, 174 insertions(+), 25 deletions(-) diff --git a/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java index 6665b7cb18..6c1d5df0be 100644 --- a/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java +++ b/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java @@ -19,9 +19,11 @@ package org.finos.waltz.common; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.stream.Collectors; @@ -132,6 +134,26 @@ public static Map> groupBy(Function keyFn, } + public static TreeMap> orderedGroupBy(Collection xs, + Function keyFn, + Function valueFn, + Comparator comparator) { + checkNotNull(xs, "xs cannot be null"); + checkNotNull(keyFn, "keyFn cannot be null"); + checkNotNull(valueFn, "valueFn cannot be null"); + + TreeMap> result = new TreeMap<>(comparator); + + for (V v: xs) { + K key = keyFn.apply(v); + Collection bucket = result.computeIfAbsent(key, u -> ListUtilities.newArrayList()); + bucket.add(valueFn.apply(v)); + result.put(key, bucket); + } + return result; + } + + public static Map indexBy(Collection xs, Function keyFn) { return indexBy(keyFn, xs); diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java index 3e586f1ee9..fea1a9a7b3 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java @@ -62,6 +62,7 @@ public static void main(String[] args) { EntityReference fdw = EntityReference.mkRef(EntityKind.APPLICATION, 16831L); EntityReference rftEds = EntityReference.mkRef(EntityKind.ORG_UNIT, 10526L); EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 95L); + EntityReference flow = EntityReference.mkRef(EntityKind.LOGICAL_DATA_FLOW, 75503L); Select> flowSelector = mkSelector(group); EntityHierarchy dtHierarchy = time("loading dt hier", () -> ehSvc.fetchHierarchyForKind(EntityKind.DATA_TYPE)); diff --git a/waltz-service/src/main/java/org/finos/waltz/service/data_type/DataTypeDecoratorService.java b/waltz-service/src/main/java/org/finos/waltz/service/data_type/DataTypeDecoratorService.java index ce5e7ed800..c1dc914ead 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/data_type/DataTypeDecoratorService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/data_type/DataTypeDecoratorService.java @@ -221,10 +221,10 @@ public int[] addDecorators(String userName, entityReference, userName); if (entityReference.kind().equals(LOGICAL_DATA_FLOW)) { - int rulesUpdated = FunctionUtilities.time("recalc ratings", () -> flowClassificationRuleService.recalculateFlowRatingsForSelector(mkOpts(entityReference))); + flowClassificationRuleService.recalculateFlowRatingsForSelector(mkOpts(entityReference)); } - FunctionUtilities.time("recalc usage", ()->recalculateDataTypeUsageForApplications(entityReference)); + recalculateDataTypeUsageForApplications(entityReference); if (PHYSICAL_SPECIFICATION.equals(entityReference.kind())) { physicalSpecificationService.propagateDataTypesToLogicalFlows(userName, entityReference.id()); diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java index b53983a0d4..46cd3ab483 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleService.java @@ -322,17 +322,20 @@ private Set lfdIdToInboundRuleIdMap.containsKey(d.lfdId()) || lfdIdToOutboundRuleIdMap.containsKey(d.lfdId())) .map(d -> { + + // If flow in scope of this population but no flow classification rules are influencing a rating we should update the decorator to the default ratings Tuple2 outboundFlowRating = lfdIdToOutboundRuleIdMap.getOrDefault(d.lfdId(), defaultOutcome); Tuple2 inboundFlowRating = lfdIdToInboundRuleIdMap.getOrDefault(d.lfdId(), defaultOutcome); - String outboundRatingCode = outboundRatingCodeByRuleId.get(outboundFlowRating.v1); - String inboundRatingCode = inboundRatingCodeByRuleId.get(inboundFlowRating.v1); + String outboundRatingCode = outboundRatingCodeByRuleId.getOrDefault(outboundFlowRating.v1, NO_OPINION.value()); + String inboundRatingCode = inboundRatingCodeByRuleId.getOrDefault(inboundFlowRating.v1, NO_OPINION.value()); AuthoritativenessRatingValue outboundRating = FlowClassificationRuleUtilities.MatchOutcome.POSITIVE_MATCH.equals(outboundFlowRating.v2) ? AuthoritativenessRatingValue.of(outboundRatingCode) - : DISCOURAGED; + : FlowClassificationRuleUtilities.MatchOutcome.NEGATIVE_MATCH.equals(outboundFlowRating.v2) + ? DISCOURAGED + : NO_OPINION; AuthoritativenessRatingValue inboundRating = FlowClassificationRuleUtilities.MatchOutcome.POSITIVE_MATCH.equals(inboundFlowRating.v2) ? AuthoritativenessRatingValue.of(inboundRatingCode) diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java index 6fcc6a11c8..b687f7d44c 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java @@ -6,18 +6,24 @@ import org.finos.waltz.model.datatype.FlowDataType; import org.finos.waltz.model.entity_hierarchy.EntityHierarchy; import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleVantagePoint; +import org.immutables.value.Value; +import org.jooq.lambda.function.Function2; import org.jooq.lambda.function.Function4; import org.jooq.lambda.tuple.Tuple2; +import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; +import static org.finos.waltz.common.MapUtilities.orderedGroupBy; import static org.jooq.lambda.tuple.Tuple.tuple; public class FlowClassificationRuleUtilities { @@ -28,6 +34,7 @@ public class FlowClassificationRuleUtilities { public static final ToLongFunction vantagePointIdComparator = d -> d.vantagePoint().id(); public static final ToLongFunction dataTypeIdComparator = FlowClassificationRuleVantagePoint::dataTypeId; public static final ToLongFunction subjectIdComparator = d -> d.subjectReference().id(); + public static final Comparator flowClassificationRuleVantagePointComparator = Comparator .comparing(kindComparator) .thenComparing(vantageComparator) @@ -43,7 +50,7 @@ protected static Map> applyVantagePoints(FlowDi EntityHierarchy ouHierarchy, EntityHierarchy dtHierarchy) { - Function4, Set, FlowDataType, MatchOutcome> matcher = determineMatcherFn(direction); + Function2 matcher = determineMatcherFn(direction); Map> lfdIdToRuleAndOutcomeMap = new HashMap<>(); @@ -57,25 +64,45 @@ protected static Map> applyVantagePoints(FlowDi .filter(rvp -> rvp.dataTypeId() == null || ruleDataTypes.contains(rvp.dataTypeId())) .collect(Collectors.toList()); - filteredRules - .forEach(rvp -> { - Set childOUs = ouHierarchy.findChildren(rvp.vantagePoint().id()); - Set childDTs = dtHierarchy.findChildren(rvp.dataTypeId()); + TreeMap> bucketedRules = bucketRules(filteredRules); + + bucketedRules + .entrySet() + .forEach(kv -> { + + BucketKey bucketKey = kv.getKey(); + Collection rvps = kv.getValue(); + + Set childOUs = ouHierarchy.findChildren(bucketKey.vantagePoint().id()); + Set childDTs = dtHierarchy.findChildren(bucketKey.dataTypeId()); + + Predicate bucketMatcher = mkBucketMatcher(direction, bucketKey, childOUs, childDTs); + population.forEach(p -> { - Tuple2 currentRuleAndOutcome = lfdIdToRuleAndOutcomeMap.get(p.lfdId()); - if (currentRuleAndOutcome != null && currentRuleAndOutcome.v2 == MatchOutcome.POSITIVE_MATCH) { + + //Can skip the entire population if another bucket has resolved the decorator + Tuple2 previousOutcome = lfdIdToRuleAndOutcomeMap.get(p.lfdId()); + if (previousOutcome != null && previousOutcome.v2 == MatchOutcome.POSITIVE_MATCH) { return; // skip, already got a good match } - MatchOutcome outcome = matcher.apply(rvp, childOUs, childDTs, p); - if (outcome == MatchOutcome.NOT_APPLICABLE) { - // skip - } else if (currentRuleAndOutcome == null) { - lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), outcome)); - } else if (currentRuleAndOutcome.v2 == MatchOutcome.NEGATIVE_MATCH && outcome == MatchOutcome.POSITIVE_MATCH) { - // override result as we have a positive match - lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), MatchOutcome.POSITIVE_MATCH)); - } else { - // skip, leave the map alone as a more specific negative rule id already exists + + if (bucketMatcher.test(p)) { + rvps + .forEach(rvp -> { + Tuple2 currentRuleAndOutcome = lfdIdToRuleAndOutcomeMap.get(p.lfdId()); + if (currentRuleAndOutcome != null && currentRuleAndOutcome.v2 == MatchOutcome.POSITIVE_MATCH) { + return; // skip, already got a good match + } + MatchOutcome outcome = matcher.apply(rvp, p); + if (currentRuleAndOutcome == null) { + lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), outcome)); + } else if (currentRuleAndOutcome.v2 == MatchOutcome.NEGATIVE_MATCH && outcome == MatchOutcome.POSITIVE_MATCH) { + // override result as we have a positive match + lfdIdToRuleAndOutcomeMap.put(p.lfdId(), tuple(rvp.ruleId(), MatchOutcome.POSITIVE_MATCH)); + } else { + // skip, leave the map alone as a more specific negative rule id already exists + } + }); } }); }); @@ -83,6 +110,62 @@ protected static Map> applyVantagePoints(FlowDi return lfdIdToRuleAndOutcomeMap; } + private static Predicate mkBucketMatcher(FlowDirection direction, BucketKey bucketKey, Set childOUs, Set childDTs) { + + if (direction.equals(FlowDirection.INBOUND)) { + return p -> { + boolean dtMatches = bucketKey.dataTypeId() == null || childDTs.contains(p.dtId()); + return dtMatches && checkScopeMatches(bucketKey, childOUs, p.source(), p.sourceOuId()); + }; + } else { + return p -> { + boolean dtMatches = bucketKey.dataTypeId() == null || childDTs.contains(p.dtId()); + return dtMatches && checkScopeMatches(bucketKey, childOUs, p.target(), p.targetOuId()); + }; + } + } + + + private static TreeMap> bucketRules(List targetedPopRules) { + + + Comparator kindComparator = Comparator.comparing(t -> t.vantagePoint().kind().name()); + Comparator vpRankComparator = Comparator.comparingInt(BucketKey::vantagePointRank); + Comparator dtRankComparator = Comparator.comparingInt(BucketKey::dataTypeRank); + + TreeMap> groups = orderedGroupBy( + targetedPopRules, + d -> ImmutableBucketKey.builder() + .vantagePoint(d.vantagePoint()) + .dataTypeId(d.dataTypeId()) + .vantagePointRank(d.vantagePointRank()) + .dataTypeRank(d.dataTypeRank()) + .build(), + d -> d, + kindComparator + .thenComparing(vpRankComparator.reversed()) + .thenComparing(dtRankComparator.reversed()) + .thenComparingLong(d -> d.vantagePoint().id()) + .thenComparingLong(BucketKey::dataTypeId)); + + System.out.println(groups.size()); + return groups; + } + + @Value.Immutable + public interface BucketKey { + + EntityReference vantagePoint(); + Long dataTypeId(); + + @Value.Auxiliary + Integer vantagePointRank(); + + @Value.Auxiliary + Integer dataTypeRank(); + + } + enum MatchOutcome { NOT_APPLICABLE, NEGATIVE_MATCH, @@ -90,7 +173,34 @@ enum MatchOutcome { } - private static Function4, Set, FlowDataType, MatchOutcome> determineMatcherFn(FlowDirection direction) { + private static Function2 determineMatcherFn(FlowDirection direction) { + Function2 inboundMatcher = + (rvp, p) -> { + boolean subjectMatches = p.target().equals(rvp.subjectReference()); + if (subjectMatches) { + return MatchOutcome.POSITIVE_MATCH; + } else { + return MatchOutcome.NEGATIVE_MATCH; + } + }; + + Function2 outboundMatcher = + (rvp, p) -> { + boolean subjectMatches = p.source().equals(rvp.subjectReference()); + if (subjectMatches) { + return MatchOutcome.POSITIVE_MATCH; + } else { + return MatchOutcome.NEGATIVE_MATCH; + } + }; + + return direction == FlowDirection.INBOUND + ? inboundMatcher + : outboundMatcher; + } + + + private static Function4, Set, FlowDataType, MatchOutcome> determineMatcherFnOld(FlowDirection direction) { Function4, Set, FlowDataType, MatchOutcome> inboundMatcher = (rvp, childOUs, childDTs, p) -> { boolean subjectMatches = p.target().equals(rvp.subjectReference()); @@ -114,7 +224,8 @@ private static Function4, Set childOUs, - EntityReference scopeEntity, Long scopeEntityOuId) { + EntityReference scopeEntity, + Long scopeEntityOuId) { if (rvp.vantagePoint().kind() == EntityKind.ORG_UNIT) { return scopeEntityOuId != null && childOUs.contains(scopeEntityOuId); } else { @@ -123,6 +234,18 @@ private static boolean checkScopeMatches(FlowClassificationRuleVantagePoint rvp, } } + private static boolean checkScopeMatches(BucketKey bucketKey, + Set childOUs, + EntityReference scopeEntity, + Long scopeEntityOuId) { + if (bucketKey.vantagePoint().kind() == EntityKind.ORG_UNIT) { + return scopeEntityOuId != null && childOUs.contains(scopeEntityOuId); + } else { + // point-to-point flows e.g. ACTOR or APPLICATION + return scopeEntity.equals(bucketKey.vantagePoint()); + } + } + private static MatchOutcome determineOutcome(boolean subjectMatches, boolean dtAndOuMatches) { From b4a62d6dbaa75ba5221b54dbd94154eabb619390 Mon Sep 17 00:00:00 2001 From: woodjes Date: Wed, 8 May 2024 17:33:50 +0100 Subject: [PATCH 3/8] Improve performance of the fcr outcomes #CTCTOWALTZ-3180 #7068 --- .../source-and-target-panel.html | 7 +++--- .../source-and-target-panel.js | 22 +++++++++++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html index d984a60d7f..b66b669942 100644 --- a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html +++ b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html @@ -156,9 +156,10 @@

  • -
    -
    + + { vm.flowClassifications = r; - vm.flowClassificationsByCode = _ + + vm.inboundClassificationsByCode = _ .chain(r) - .filter(d => d.direction === vm.ratingDirection) - .keyBy( d => d.code) + .filter(d => d.direction === FlowDirection.INBOUND.key) + .keyBy(d => d.code) .value(); + vm.outboundClassificationsByCode = _ + .chain(r) + .filter(d => d.direction === FlowDirection.OUTBOUND.key) + .keyBy(d => d.code) + .value() + + vm.flowClassificationsByCode = vm.ratingDirection === FlowDirection.INBOUND.key + ? vm.inboundClassificationsByCode + : vm.outboundClassificationsByCode; }); serviceBroker From f4e35298ee7db43f9c493722638b913f538c6199 Mon Sep 17 00:00:00 2001 From: woodjes Date: Thu, 9 May 2024 09:03:09 +0100 Subject: [PATCH 4/8] Small UI fixes - producer/consumer ratings on popover #CTCTOWALTZ-3180 #7068 --- .../source-and-target-panel/source-and-target-panel.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html index b66b669942..db0e89bf4f 100644 --- a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html +++ b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.html @@ -165,7 +165,7 @@

    name: type.name}" icon-placement="none" tooltip-placement="top" - additional-display-data="[{name: 'Rating', value: $ctrl.flowClassificationsByCode[type.rating].name}]"> + additional-display-data="[{name: 'Producer Rating', value: $ctrl.flowClassificationsByCode[type.rating].name}, {name: 'Consumer Rating', value: $ctrl.flowClassificationsByCode[type.inboundRating].name}]">

From b9b61d2b9fb5b108a9801ef7d2ff3fb0c9c32035 Mon Sep 17 00:00:00 2001 From: woodjes Date: Thu, 9 May 2024 11:39:26 +0100 Subject: [PATCH 5/8] Small UI fixes - producer/consumer ratings on popover #CTCTOWALTZ-3180 #7068 --- .../inline-logical-flow-panel.html | 17 ++++++++-- .../inline-logical-flow-panel.js | 31 +++++++++++++------ .../source-and-target-panel.js | 14 +++++++-- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html b/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html index 5c4a6cb192..9dd4a61304 100644 --- a/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html +++ b/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html @@ -21,15 +21,28 @@ logical-flows="$ctrl.logicalFlows" decorators="$ctrl.logicalFlowDecorators" physical-flows="$ctrl.physicalFlows" - physical-specifications="$ctrl.physicalSpecifications"> + physical-specifications="$ctrl.physicalSpecifications" + rating-direction="$ctrl.ratingDirection"> + + +

- Diagram detailing the applications data types (Middle) + Diagram detailing the application's data types (Middle) from the sending system (LHS) to the receiving system on the (RHS). Red lines indicate non-strategic flows, Amber indicates secondary and Green indicates a primary system.

+ +

+ + +

\ No newline at end of file diff --git a/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js b/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js index 8b7e8f1c65..ca7ffd39d0 100644 --- a/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js +++ b/waltz-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js @@ -16,15 +16,17 @@ * */ -import {CORE_API} from '../../../common/services/core-api-utils'; -import {initialiseData} from '../../../common'; +import {CORE_API} from "../../../common/services/core-api-utils"; +import {initialiseData} from "../../../common"; -import template from './inline-logical-flow-panel.html'; +import template from "./inline-logical-flow-panel.html"; import {entity} from "../../../common/services/enums/entity"; +import {flowDirection, flowDirection as FlowDirection} from "../../../common/services/enums/flow-direction"; +import FlowClassificationLegend from "../../../flow-classification-rule/components/svelte/FlowClassificationLegend.svelte" const bindings = { - parentEntityRef: '<' + parentEntityRef: "<" }; @@ -32,7 +34,9 @@ const initialState = { logicalFlows: [], logicalFlowDecorators: [], physicalFlows: [], - physicalSpecifications: [] + physicalSpecifications: [], + ratingDirection: FlowDirection.OUTBOUND.key, + FlowClassificationLegend }; @@ -40,10 +44,10 @@ function controller(serviceBroker) { const vm = initialiseData(this, initialState); - vm.$onInit = () => { + vm.$onChanges = () => { const selector = { entityReference: vm.parentEntityRef, - scope: 'EXACT' + scope: "EXACT" }; serviceBroker @@ -70,11 +74,20 @@ function controller(serviceBroker) { [vm.parentEntityRef]) .then(r => vm.physicalSpecifications = r.data); }; + + + vm.onToggleRatingDirection = () => { + if(vm.ratingDirection === flowDirection.OUTBOUND.key) { + vm.ratingDirection = flowDirection.INBOUND.key; + } else { + vm.ratingDirection = flowDirection.OUTBOUND.key; + } + } } controller.$inject = [ - 'ServiceBroker' + "ServiceBroker" ]; @@ -87,5 +100,5 @@ const component = { export default { component, - id: 'waltzInlineLogicalFlowPanel' + id: "waltzInlineLogicalFlowPanel" }; diff --git a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.js b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.js index 919e0a9b58..926154aca2 100644 --- a/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.js +++ b/waltz-ng/client/logical-flow/components/source-and-target-panel/source-and-target-panel.js @@ -258,11 +258,21 @@ function controller($element, loadFlowClassificationRatings(serviceBroker) .then(r => { vm.flowClassifications = r; - vm.flowClassificationsByCode = _ + + vm.inboundClassificationsByCode = _ .chain(r) - .filter(d => d.direction === vm.ratingDirection) + .filter(d => d.direction === FlowDirection.INBOUND.key) .keyBy(d => d.code) .value(); + vm.outboundClassificationsByCode = _ + .chain(r) + .filter(d => d.direction === FlowDirection.OUTBOUND.key) + .keyBy(d => d.code) + .value() + + vm.flowClassificationsByCode = vm.ratingDirection === FlowDirection.INBOUND.key + ? vm.inboundClassificationsByCode + : vm.outboundClassificationsByCode; }); if (changes.logicalFlows || changes.decorators) { From 9e9cdcfca3826d2766028c6eccc90bd36c91b22c Mon Sep 17 00:00:00 2001 From: "david.watkins@db.com" Date: Thu, 9 May 2024 11:48:54 +0100 Subject: [PATCH 6/8] Flow Classification re-eval on rule change #CTCTOWALTZ-3177 --- .../LogicalFlowDecoratorDao.java | 15 +++++++++++---- .../FlowClassificationRuleDao.java | 2 +- .../harness/FlowClassificationRuleHarness.java | 2 ++ .../FlowClassificationRuleUtilities.java | 9 ++++++--- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/waltz-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java b/waltz-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java index c91db2e4fa..a27aa818c0 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java @@ -32,6 +32,7 @@ import org.finos.waltz.model.rating.AuthoritativenessRatingValue; import org.finos.waltz.schema.Tables; import org.finos.waltz.schema.tables.Application; +import org.finos.waltz.schema.tables.EndUserApplication; import org.finos.waltz.schema.tables.EntityHierarchy; import org.finos.waltz.schema.tables.LogicalFlow; import org.finos.waltz.schema.tables.LogicalFlowDecorator; @@ -76,6 +77,8 @@ public class LogicalFlowDecoratorDao extends DataTypeDecoratorDao { private static final LogicalFlowDecorator lfd = Tables.LOGICAL_FLOW_DECORATOR; private static final Application srcApp = Tables.APPLICATION.as("srcApp"); private static final Application targetApp = Tables.APPLICATION.as("targetApp"); + private static final EndUserApplication srcEuda = Tables.END_USER_APPLICATION.as("srcEuda"); + private static final EndUserApplication targetEuda = Tables.END_USER_APPLICATION.as("targetEuda"); private static final EntityHierarchy eh = Tables.ENTITY_HIERARCHY; private static final Field ENTITY_NAME_FIELD = InlineSelectFieldFactory.mkNameField( @@ -353,6 +356,8 @@ public Set fetchFlowDataTypePopulationForFlowSelector(Select fetchFlowDataTypePopulation(Condition condition) { + Field srcOU = DSL.coalesce(srcApp.ORGANISATIONAL_UNIT_ID, srcEuda.ORGANISATIONAL_UNIT_ID).as("srcOU"); + Field targetOU = DSL.coalesce(targetApp.ORGANISATIONAL_UNIT_ID, targetEuda.ORGANISATIONAL_UNIT_ID).as("targetOU"); return dsl .select(lf.ID, lfd.ID, @@ -363,14 +368,16 @@ public Set fetchFlowDataTypePopulation(Condition condition) { lf.SOURCE_ENTITY_KIND, lf.TARGET_ENTITY_ID, lf.TARGET_ENTITY_KIND, - srcApp.ORGANISATIONAL_UNIT_ID, - targetApp.ORGANISATIONAL_UNIT_ID, + srcOU, + targetOU, lfd.RATING, lfd.TARGET_INBOUND_RATING) .from(lf) .innerJoin(lfd).on(lfd.LOGICAL_FLOW_ID.eq(lf.ID).and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name()))) .leftJoin(srcApp).on(srcApp.ID.eq(lf.SOURCE_ENTITY_ID).and(lf.SOURCE_ENTITY_KIND.eq(EntityKind.APPLICATION.name()))) .leftJoin(targetApp).on(targetApp.ID.eq(lf.TARGET_ENTITY_ID).and(lf.TARGET_ENTITY_KIND.eq(EntityKind.APPLICATION.name()))) + .leftJoin(srcEuda).on(srcEuda.ID.eq(lf.SOURCE_ENTITY_ID).and(lf.SOURCE_ENTITY_KIND.eq(EntityKind.END_USER_APPLICATION.name()))) + .leftJoin(targetEuda).on(targetEuda.ID.eq(lf.TARGET_ENTITY_ID).and(lf.TARGET_ENTITY_KIND.eq(EntityKind.END_USER_APPLICATION.name()))) .where(lf.IS_REMOVED.isFalse() .and(lf.ENTITY_LIFECYCLE_STATUS.eq(EntityLifecycleStatus.ACTIVE.name()))) .and(condition) @@ -383,8 +390,8 @@ public Set fetchFlowDataTypePopulation(Condition condition) { .target(readRef(r, lf.TARGET_ENTITY_KIND, lf.TARGET_ENTITY_ID)) .inboundRuleId(r.get(lfd.INBOUND_FLOW_CLASSIFICATION_RULE_ID)) .outboundRuleId(r.get(lfd.FLOW_CLASSIFICATION_RULE_ID)) - .sourceOuId(r.get(srcApp.ORGANISATIONAL_UNIT_ID)) - .targetOuId(r.get(targetApp.ORGANISATIONAL_UNIT_ID)) + .sourceOuId(r.get(srcOU)) + .targetOuId(r.get(targetOU)) .sourceOutboundRating(AuthoritativenessRatingValue.of(r.get(lfd.RATING))) .targetInboundRating(AuthoritativenessRatingValue.of(r.get(lfd.TARGET_INBOUND_RATING))) .build()); diff --git a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java index 2d1f54a8c9..7a9a1d977c 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java @@ -419,7 +419,7 @@ public List findFlowClassificationRuleVantag Set dataTypeIdsToConsider) { return findFlowClassificationRuleVantagePoints( FLOW_CLASSIFICATION.DIRECTION.eq(direction.name()) - .and(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.in(dataTypeIdsToConsider))); + .and(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.in(dataTypeIdsToConsider).or(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.isNull()))); } diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java index fea1a9a7b3..786618ba2f 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java @@ -63,6 +63,7 @@ public static void main(String[] args) { EntityReference rftEds = EntityReference.mkRef(EntityKind.ORG_UNIT, 10526L); EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 95L); EntityReference flow = EntityReference.mkRef(EntityKind.LOGICAL_DATA_FLOW, 75503L); + EntityReference tokyo = EntityReference.mkRef(EntityKind.END_USER_APPLICATION, 11104L); Select> flowSelector = mkSelector(group); EntityHierarchy dtHierarchy = time("loading dt hier", () -> ehSvc.fetchHierarchyForKind(EntityKind.DATA_TYPE)); @@ -77,6 +78,7 @@ public static void main(String[] args) { List allRules = time("fcrDao::findRules (all)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND)); List targetedDtRules = time("fcrDao::findRules (targeted - dt)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, possibleDtIds)); List targetedPopRules = time("fcrDao::findRules (targeted- pop)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, dtHierarchy, population)); + List targetedInboundPopRules = time("fcrDao::findRules (targeted- inbound pop)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.INBOUND, dtHierarchy, population)); System.out.printf( "\nRules: pop:%d / dt:%d / all:%d, Population: %d, DTs: %d\n\n", diff --git a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java index b687f7d44c..454b44e30f 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/flow_classification_rule/FlowClassificationRuleUtilities.java @@ -3,6 +3,7 @@ import org.finos.waltz.model.EntityKind; import org.finos.waltz.model.EntityReference; import org.finos.waltz.model.FlowDirection; +import org.finos.waltz.model.Nullable; import org.finos.waltz.model.datatype.FlowDataType; import org.finos.waltz.model.entity_hierarchy.EntityHierarchy; import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleVantagePoint; @@ -71,7 +72,6 @@ protected static Map> applyVantagePoints(FlowDi .forEach(kv -> { BucketKey bucketKey = kv.getKey(); - Collection rvps = kv.getValue(); Set childOUs = ouHierarchy.findChildren(bucketKey.vantagePoint().id()); Set childDTs = dtHierarchy.findChildren(bucketKey.dataTypeId()); @@ -86,7 +86,9 @@ protected static Map> applyVantagePoints(FlowDi return; // skip, already got a good match } + // System.out.printf("Testing bucketKey: %s against flow: %s - outcome: %s\n", bucketKey, p, bucketMatcher.test(p)); if (bucketMatcher.test(p)) { + Collection rvps = kv.getValue(); rvps .forEach(rvp -> { Tuple2 currentRuleAndOutcome = lfdIdToRuleAndOutcomeMap.get(p.lfdId()); @@ -146,9 +148,8 @@ private static TreeMap .thenComparing(vpRankComparator.reversed()) .thenComparing(dtRankComparator.reversed()) .thenComparingLong(d -> d.vantagePoint().id()) - .thenComparingLong(BucketKey::dataTypeId)); + .thenComparingLong(d -> d.dataTypeId() == null ? Long.MAX_VALUE : d.dataTypeId())); - System.out.println(groups.size()); return groups; } @@ -156,6 +157,8 @@ private static TreeMap public interface BucketKey { EntityReference vantagePoint(); + + @Nullable Long dataTypeId(); @Value.Auxiliary From 8137cbb0b86b5f76bd98246215d3992f4e8d2dec Mon Sep 17 00:00:00 2001 From: woodjes Date: Thu, 9 May 2024 14:42:45 +0100 Subject: [PATCH 7/8] Small UI fixes - producer/consumer ratings on flow details #CTCTOWALTZ-3180 #7068 --- .../flow-detail-tab/DataTypeMiniTable.svelte | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/waltz-ng/client/data-flow/components/svelte/flow-detail-tab/DataTypeMiniTable.svelte b/waltz-ng/client/data-flow/components/svelte/flow-detail-tab/DataTypeMiniTable.svelte index ce14030323..885901cabf 100644 --- a/waltz-ng/client/data-flow/components/svelte/flow-detail-tab/DataTypeMiniTable.svelte +++ b/waltz-ng/client/data-flow/components/svelte/flow-detail-tab/DataTypeMiniTable.svelte @@ -7,12 +7,19 @@ import _ from "lodash"; import EntityLabel from "../../../../common/svelte/EntityLabel.svelte"; import Toggle from "../../../../common/svelte/Toggle.svelte"; + import FlowRatingCell from "../../../../common/svelte/FlowRatingCell.svelte"; + import {flowDirection as FlowDirection} from "../../../../common/services/enums/flow-direction"; + import RatingIndicatorCell from "../../../../ratings/components/rating-indicator-cell/RatingIndicatorCell.svelte"; export let decorators = []; export let flowClassifications = [] + let inboundClassifications; + let outboundClassifications; - $: flowClassificationsByCode = _.keyBy(flowClassifications, d => d.code); + $: [inboundClassifications, outboundClassifications] = _.partition(flowClassifications, d => d.direction === FlowDirection.INBOUND.key); + $: inboundClassificationsByCode = _.keyBy(inboundClassifications, d => d.code); + $: outboundClassificationsByCode = _.keyBy(outboundClassifications, d => d.code); $: hasRatings = !_.isEmpty(flowClassifications) $: showRatings = !$hideRatings && hasRatings; @@ -20,25 +27,39 @@ - {#each _.orderBy(decorators, d => d.decoratorEntity.name) as type} + - - {#if showRatings} + + {#if showRatings} + + - {/if} - + + + + + + {/if} + {/each} -
-
-
+ 		+
+ Producer Rating + - {_.get(flowClassificationsByCode, [type.rating, "name"], "-")} +
+ Consumer Rating + + +
{#if hasRatings} @@ -54,14 +75,6 @@ \ No newline at end of file From 873509793e6014dfa62b2a7087ce9db400a97d86 Mon Sep 17 00:00:00 2001 From: "david.watkins@db.com" Date: Thu, 9 May 2024 17:48:56 +0100 Subject: [PATCH 8/8] Flow Classification re-eval on rule change #CTCTOWALTZ-3177 --- .../org/finos/waltz/data/CommonTableFieldsRegistry.java | 2 +- .../waltz/jobs/harness/FlowClassificationRuleHarness.java | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java b/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java index 484b27a799..858f96ee06 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java @@ -427,7 +427,7 @@ public static CommonTableFields determineCommonTableFields(EntityKind kind, S .table(sq) .idField(sq.ID) .parentIdField(null) - .nameField(sq.LABEL) + .nameField(sq.QUESTION_TEXT) .descriptionField(sq.HELP_TEXT) .externalIdField(sq.EXTERNAL_ID) .build(); diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java index 786618ba2f..99d83b7e91 100644 --- a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java +++ b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java @@ -58,12 +58,7 @@ public static void main(String[] args) { FlowClassificationRuleDao fcrDao = ctx.getBean(FlowClassificationRuleDao.class); LogicalFlowDecoratorDao lfdDao = ctx.getBean(LogicalFlowDecoratorDao.class); - EntityReference collibra = EntityReference.mkRef(EntityKind.APPLICATION, 124334001L); - EntityReference fdw = EntityReference.mkRef(EntityKind.APPLICATION, 16831L); - EntityReference rftEds = EntityReference.mkRef(EntityKind.ORG_UNIT, 10526L); - EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 95L); - EntityReference flow = EntityReference.mkRef(EntityKind.LOGICAL_DATA_FLOW, 75503L); - EntityReference tokyo = EntityReference.mkRef(EntityKind.END_USER_APPLICATION, 11104L); + EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 99L); Select> flowSelector = mkSelector(group); EntityHierarchy dtHierarchy = time("loading dt hier", () -> ehSvc.fetchHierarchyForKind(EntityKind.DATA_TYPE));