Pull request #376: Flow Classification recalc optimising plus small ui …

…fixes #7067 Merge in WALTZ/waltz from WALTZ/waltz-dw:CTCTOWALTZ-3177-fcr-recalc-3-7067 to db-feature/waltz-7068-small-ui-fixes-fcrs * commit '873509793e6014dfa62b2a7087ce9db400a97d86': Flow Classification re-eval on rule change #CTCTOWALTZ-3177 Small UI fixes Flow Classification re-eval on rule change #CTCTOWALTZ-3177 Small UI fixes Small UI fixes Improve performance of the fcr outcomes Improve performance of the fcr outcomes Flow Classification re-eval on rule change #CTCTOWALTZ-3177
finos · May 13, 2024 · 04eeffa · 04eeffa
2 parents b18626f + 8735097
commit 04eeffa
Show file tree

Hide file tree

Showing 14 changed files with 409 additions and 109 deletions.
diff --git a/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/FunctionUtilities.java
@@ -66,10 +66,11 @@ public static <T> T time(String name, Supplier<T> supplier) {
         long st = System.currentTimeMillis();
 
         try {
+            LOG.info("----> begin [{}]", name);
             T r = supplier.get();
             long end = System.currentTimeMillis();
 
-            LOG.info("duration [{}]: {}", name, (end - st));
+            LOG.info("<---- end [{}], duration:{}", name, (end - st));
             return r;
         } catch (Exception e) {
             String msg = String.format("Unexpected error when timing [%s]: %s", name, e.getMessage());

diff --git a/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java b/waltz-common/src/main/java/org/finos/waltz/common/MapUtilities.java
@@ -19,9 +19,11 @@
 package org.finos.waltz.common;
 
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.TreeMap;
 import java.util.function.BinaryOperator;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -132,6 +134,26 @@ public static <K, V, V2> Map<K, Collection<V2>> groupBy(Function<V, K> keyFn,
     }
 
 
+    public static <K, V, V2> TreeMap<K, Collection<V2>> orderedGroupBy(Collection<V> xs,
+                                                                       Function<V, K> keyFn,
+                                                                       Function<V, V2> valueFn,
+                                                                       Comparator<K> comparator) {
+        checkNotNull(xs, "xs cannot be null");
+        checkNotNull(keyFn, "keyFn cannot be null");
+        checkNotNull(valueFn, "valueFn cannot be null");
+
+        TreeMap<K, Collection<V2>> result = new TreeMap<>(comparator);
+
+        for (V v: xs) {
+            K key = keyFn.apply(v);
+            Collection<V2> bucket = result.computeIfAbsent(key, u -> ListUtilities.newArrayList());
+            bucket.add(valueFn.apply(v));
+            result.put(key, bucket);
+        }
+        return result;
+    }
+
+
     public static <K, V> Map<K, V> indexBy(Collection<V> xs,
                                            Function<V, K> keyFn) {
         return indexBy(keyFn, xs);

diff --git a/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java b/waltz-data/src/main/java/org/finos/waltz/data/CommonTableFieldsRegistry.java
@@ -427,7 +427,7 @@ public static CommonTableFields<?> determineCommonTableFields(EntityKind kind, S
                         .table(sq)
                         .idField(sq.ID)
                         .parentIdField(null)
-                        .nameField(sq.LABEL)
+                        .nameField(sq.QUESTION_TEXT)
                         .descriptionField(sq.HELP_TEXT)
                         .externalIdField(sq.EXTERNAL_ID)
                         .build();

diff --git a/...z-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java b/...z-data/src/main/java/org/finos/waltz/data/datatype_decorator/LogicalFlowDecoratorDao.java
@@ -32,6 +32,7 @@
 import org.finos.waltz.model.rating.AuthoritativenessRatingValue;
 import org.finos.waltz.schema.Tables;
 import org.finos.waltz.schema.tables.Application;
+import org.finos.waltz.schema.tables.EndUserApplication;
 import org.finos.waltz.schema.tables.EntityHierarchy;
 import org.finos.waltz.schema.tables.LogicalFlow;
 import org.finos.waltz.schema.tables.LogicalFlowDecorator;
@@ -76,6 +77,8 @@ public class LogicalFlowDecoratorDao extends DataTypeDecoratorDao {
     private static final LogicalFlowDecorator lfd = Tables.LOGICAL_FLOW_DECORATOR;
     private static final Application srcApp = Tables.APPLICATION.as("srcApp");
     private static final Application targetApp = Tables.APPLICATION.as("targetApp");
+    private static final EndUserApplication srcEuda = Tables.END_USER_APPLICATION.as("srcEuda");
+    private static final EndUserApplication targetEuda = Tables.END_USER_APPLICATION.as("targetEuda");
     private static final EntityHierarchy eh = Tables.ENTITY_HIERARCHY;
 
     private static final Field<String> ENTITY_NAME_FIELD = InlineSelectFieldFactory.mkNameField(
@@ -353,6 +356,8 @@ public Set<FlowDataType> fetchFlowDataTypePopulationForFlowSelector(Select<Recor
     }
 
     public Set<FlowDataType> fetchFlowDataTypePopulation(Condition condition) {
+        Field<Long> srcOU = DSL.coalesce(srcApp.ORGANISATIONAL_UNIT_ID, srcEuda.ORGANISATIONAL_UNIT_ID).as("srcOU");
+        Field<Long> targetOU = DSL.coalesce(targetApp.ORGANISATIONAL_UNIT_ID, targetEuda.ORGANISATIONAL_UNIT_ID).as("targetOU");
         return dsl
                 .select(lf.ID,
                         lfd.ID,
@@ -363,14 +368,16 @@ public Set<FlowDataType> fetchFlowDataTypePopulation(Condition condition) {
                         lf.SOURCE_ENTITY_KIND,
                         lf.TARGET_ENTITY_ID,
                         lf.TARGET_ENTITY_KIND,
-                        srcApp.ORGANISATIONAL_UNIT_ID,
-                        targetApp.ORGANISATIONAL_UNIT_ID,
+                        srcOU,
+                        targetOU,
                         lfd.RATING,
                         lfd.TARGET_INBOUND_RATING)
                 .from(lf)
                 .innerJoin(lfd).on(lfd.LOGICAL_FLOW_ID.eq(lf.ID).and(lfd.DECORATOR_ENTITY_KIND.eq(EntityKind.DATA_TYPE.name())))
                 .leftJoin(srcApp).on(srcApp.ID.eq(lf.SOURCE_ENTITY_ID).and(lf.SOURCE_ENTITY_KIND.eq(EntityKind.APPLICATION.name())))
                 .leftJoin(targetApp).on(targetApp.ID.eq(lf.TARGET_ENTITY_ID).and(lf.TARGET_ENTITY_KIND.eq(EntityKind.APPLICATION.name())))
+                .leftJoin(srcEuda).on(srcEuda.ID.eq(lf.SOURCE_ENTITY_ID).and(lf.SOURCE_ENTITY_KIND.eq(EntityKind.END_USER_APPLICATION.name())))
+                .leftJoin(targetEuda).on(targetEuda.ID.eq(lf.TARGET_ENTITY_ID).and(lf.TARGET_ENTITY_KIND.eq(EntityKind.END_USER_APPLICATION.name())))
                 .where(lf.IS_REMOVED.isFalse()
                         .and(lf.ENTITY_LIFECYCLE_STATUS.eq(EntityLifecycleStatus.ACTIVE.name())))
                 .and(condition)
@@ -383,8 +390,8 @@ public Set<FlowDataType> fetchFlowDataTypePopulation(Condition condition) {
                         .target(readRef(r, lf.TARGET_ENTITY_KIND, lf.TARGET_ENTITY_ID))
                         .inboundRuleId(r.get(lfd.INBOUND_FLOW_CLASSIFICATION_RULE_ID))
                         .outboundRuleId(r.get(lfd.FLOW_CLASSIFICATION_RULE_ID))
-                        .sourceOuId(r.get(srcApp.ORGANISATIONAL_UNIT_ID))
-                        .targetOuId(r.get(targetApp.ORGANISATIONAL_UNIT_ID))
+                        .sourceOuId(r.get(srcOU))
+                        .targetOuId(r.get(targetOU))
                         .sourceOutboundRating(AuthoritativenessRatingValue.of(r.get(lfd.RATING)))
                         .targetInboundRating(AuthoritativenessRatingValue.of(r.get(lfd.TARGET_INBOUND_RATING)))
                         .build());

diff --git a/...rc/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java b/...rc/main/java/org/finos/waltz/data/flow_classification_rule/FlowClassificationRuleDao.java
@@ -24,6 +24,7 @@
 import org.finos.waltz.model.FlowDirection;
 import org.finos.waltz.model.ImmutableEntityReference;
 import org.finos.waltz.model.MessageSeverity;
+import org.finos.waltz.model.datatype.FlowDataType;
 import org.finos.waltz.model.flow_classification_rule.DiscouragedSource;
 import org.finos.waltz.model.flow_classification_rule.FlowClassificationRule;
 import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleCreateCommand;
@@ -414,6 +415,28 @@ public List<FlowClassificationRuleVantagePoint> findFlowClassificationRuleVantag
     }
 
 
+    public List<FlowClassificationRuleVantagePoint> findFlowClassificationRuleVantagePoints(FlowDirection direction,
+                                                                                            Set<Long> dataTypeIdsToConsider) {
+        return findFlowClassificationRuleVantagePoints(
+                FLOW_CLASSIFICATION.DIRECTION.eq(direction.name())
+                        .and(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.in(dataTypeIdsToConsider).or(FLOW_CLASSIFICATION_RULE.DATA_TYPE_ID.isNull())));
+    }
+
+
+
+    public List<FlowClassificationRuleVantagePoint> findFlowClassificationRuleVantagePoints(FlowDirection direction,
+                                                                                            org.finos.waltz.model.entity_hierarchy.EntityHierarchy dtHierarchy,
+                                                                                            Set<FlowDataType> population) {
+        Set<Long> possibleDtIds = population
+                .stream()
+                .map(FlowDataType::dtId)
+                .distinct()
+                .flatMap(dtId -> dtHierarchy.findAncestors(dtId).stream())
+                .collect(Collectors.toSet());
+        return findFlowClassificationRuleVantagePoints(direction, possibleDtIds);
+    }
+
+
     private List<FlowClassificationRuleVantagePoint> findFlowClassificationRuleVantagePoints(Condition condition) {
         SelectSeekStep6<Record11<Long, String, Integer, Long, Integer, Long, String, String, Long, String, String>, String, Integer, Integer, Long, Long, Long> select = dsl
                 .select(vantagePointId,

diff --git a/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java b/waltz-jobs/src/main/java/org/finos/waltz/jobs/harness/FlowClassificationRuleHarness.java
@@ -18,31 +18,78 @@
 
 package org.finos.waltz.jobs.harness;
 
+import org.finos.waltz.data.datatype_decorator.LogicalFlowDecoratorDao;
+import org.finos.waltz.data.flow_classification_rule.FlowClassificationRuleDao;
+import org.finos.waltz.data.logical_flow.LogicalFlowIdSelectorFactory;
 import org.finos.waltz.model.EntityKind;
-import org.finos.waltz.model.flow_classification_rule.FlowClassificationRule;
+import org.finos.waltz.model.EntityReference;
+import org.finos.waltz.model.FlowDirection;
+import org.finos.waltz.model.IdSelectionOptions;
+import org.finos.waltz.model.datatype.FlowDataType;
+import org.finos.waltz.model.entity_hierarchy.EntityHierarchy;
+import org.finos.waltz.model.flow_classification_rule.FlowClassificationRuleVantagePoint;
 import org.finos.waltz.service.DIConfiguration;
+import org.finos.waltz.service.entity_hierarchy.EntityHierarchyService;
 import org.finos.waltz.service.flow_classification_rule.FlowClassificationRuleService;
 import org.jooq.DSLContext;
+import org.jooq.Record1;
+import org.jooq.Select;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 
+import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
-import static org.finos.waltz.model.EntityReference.mkRef;
-import static org.finos.waltz.model.IdSelectionOptions.mkOpts;
+import static org.finos.waltz.common.FunctionUtilities.time;
 
 
 public class FlowClassificationRuleHarness {
 
+    private static final LogicalFlowIdSelectorFactory logicalFlowIdSelectorFactory = new LogicalFlowIdSelectorFactory();
+
+
     public static void main(String[] args) {
         AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class);
         DSLContext dsl = ctx.getBean(DSLContext.class);
 
 
-        FlowClassificationRuleService svc = ctx.getBean(FlowClassificationRuleService.class);
-        Set<FlowClassificationRule> r = svc.findClassificationRules(mkOpts(mkRef(EntityKind.ORG_UNIT, 14902L)));
-        System.out.println(r);
+        FlowClassificationRuleService fcrSvc = ctx.getBean(FlowClassificationRuleService.class);
+        EntityHierarchyService ehSvc = ctx.getBean(EntityHierarchyService.class);
+        FlowClassificationRuleDao fcrDao = ctx.getBean(FlowClassificationRuleDao.class);
+        LogicalFlowDecoratorDao lfdDao = ctx.getBean(LogicalFlowDecoratorDao.class);
+
+        EntityReference group = EntityReference.mkRef(EntityKind.ORG_UNIT, 99L);
+
+        Select<Record1<Long>> flowSelector = mkSelector(group);
+        EntityHierarchy dtHierarchy = time("loading dt hier", () -> ehSvc.fetchHierarchyForKind(EntityKind.DATA_TYPE));
+        Set<FlowDataType> population = time("lfd::fetchPopulation", () -> lfdDao.fetchFlowDataTypePopulationForFlowSelector(flowSelector));
+        Set<Long> possibleDtIds = population
+                .stream()
+                .map(FlowDataType::dtId)
+                .distinct()
+                .flatMap(dtId -> dtHierarchy.findAncestors(dtId).stream())
+                .collect(Collectors.toSet());
+
+        List<FlowClassificationRuleVantagePoint> allRules = time("fcrDao::findRules (all)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND));
+        List<FlowClassificationRuleVantagePoint> targetedDtRules = time("fcrDao::findRules (targeted - dt)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, possibleDtIds));
+        List<FlowClassificationRuleVantagePoint> targetedPopRules = time("fcrDao::findRules (targeted- pop)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.OUTBOUND, dtHierarchy, population));
+        List<FlowClassificationRuleVantagePoint> targetedInboundPopRules = time("fcrDao::findRules (targeted- inbound pop)", () -> fcrDao.findFlowClassificationRuleVantagePoints(FlowDirection.INBOUND, dtHierarchy, population));
+
+        System.out.printf(
+                "\nRules: pop:%d / dt:%d / all:%d, Population: %d, DTs: %d\n\n",
+                targetedPopRules.size(),
+                targetedDtRules.size(),
+                allRules.size(),
+                population.size(),
+                possibleDtIds.size());
+
+        time("recalc", () -> fcrSvc.recalculateRatingsForPopulation(population));
+    }
 
-//        System.exit(-1);
+    private static Select<Record1<Long>> mkSelector(EntityReference ref) {
+        IdSelectionOptions opts = IdSelectionOptions.mkOpts(ref);
+        Select<Record1<Long>> flowSelector = logicalFlowIdSelectorFactory.apply(opts);
+        return flowSelector;
     }
 
 

diff --git a/...ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html b/...ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.html
@@ -21,15 +21,28 @@
                                    logical-flows="$ctrl.logicalFlows"
                                    decorators="$ctrl.logicalFlowDecorators"
                                    physical-flows="$ctrl.physicalFlows"
-                                   physical-specifications="$ctrl.physicalSpecifications">
+                                   physical-specifications="$ctrl.physicalSpecifications"
+                                   rating-direction="$ctrl.ratingDirection">
     </waltz-source-and-target-panel>
 
+    <waltz-toggle state="$ctrl.ratingDirection === 'OUTBOUND'"
+                  on-toggle="$ctrl.onToggleRatingDirection()"
+                  label-on="Showing producer flow ratings"
+                  label-off="Showing consumer flow ratings">
+    </waltz-toggle>
+
     <p class="text-muted small">
-        Diagram detailing the applications data types (Middle)
+        Diagram detailing the application's data types (Middle)
         from the sending system (LHS)
         to the receiving system on the (RHS).
         <span style="color: darkred">Red lines</span> indicate non-strategic flows,
         <span style="color: orange">Amber</span> indicates secondary and
         <span style="color: darkgreen">Green</span> indicates a primary system.
     </p>
+
+    <p class="help-block">
+        <waltz-svelte-component component="$ctrl.FlowClassificationLegend"
+                                rating-direction="$ctrl.ratingDirection">
+        </waltz-svelte-component>
+    </p>
 </div>
diff --git a/...z-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js b/...z-ng/client/attestation/components/inline-logical-flow-panel/inline-logical-flow-panel.js
@@ -16,34 +16,38 @@
  *
  */
 
-import {CORE_API} from '../../../common/services/core-api-utils';
-import {initialiseData} from '../../../common';
+import {CORE_API} from "../../../common/services/core-api-utils";
+import {initialiseData} from "../../../common";
 
-import template from './inline-logical-flow-panel.html';
+import template from "./inline-logical-flow-panel.html";
 import {entity} from "../../../common/services/enums/entity";
+import {flowDirection, flowDirection as FlowDirection} from "../../../common/services/enums/flow-direction";
+import FlowClassificationLegend from "../../../flow-classification-rule/components/svelte/FlowClassificationLegend.svelte"
 
 
 const bindings = {
-    parentEntityRef: '<'
+    parentEntityRef: "<"
 };
 
 
 const initialState = {
     logicalFlows: [],
     logicalFlowDecorators: [],
     physicalFlows: [],
-    physicalSpecifications: []
+    physicalSpecifications: [],
+    ratingDirection: FlowDirection.OUTBOUND.key,
+    FlowClassificationLegend
 };
 
 
 function controller(serviceBroker) {
 
     const vm = initialiseData(this, initialState);
 
-    vm.$onInit = () => {
+    vm.$onChanges = () => {
         const selector = {
             entityReference: vm.parentEntityRef,
-            scope: 'EXACT'
+            scope: "EXACT"
         };
 
         serviceBroker
@@ -70,11 +74,20 @@ function controller(serviceBroker) {
                 [vm.parentEntityRef])
             .then(r => vm.physicalSpecifications = r.data);
     };
+
+
+    vm.onToggleRatingDirection = () => {
+        if(vm.ratingDirection === flowDirection.OUTBOUND.key) {
+            vm.ratingDirection = flowDirection.INBOUND.key;
+        } else {
+            vm.ratingDirection = flowDirection.OUTBOUND.key;
+        }
+    }
 }
 
 
 controller.$inject = [
-    'ServiceBroker'
+    "ServiceBroker"
 ];
 
 
@@ -87,5 +100,5 @@ const component = {
 
 export default {
     component,
-    id: 'waltzInlineLogicalFlowPanel'
+    id: "waltzInlineLogicalFlowPanel"
 };