Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Db contrib/waltz 7138 data type merge #7150

Merged
merged 4 commits into from
Sep 24, 2024
Merged

Db contrib/waltz 7138 data type merge #7150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2316b4b
Data Type Maintain merge
Sep 17, 2024
5ffbeea
Merge branch 'CTCOWALTZ-3346-dt-merge-7138' into CTCTOWALTZ-3346-data…
Sep 17, 2024
71b5731
Data Type Maintain merge
Sep 23, 2024
878d39c
Pull request #396: CTCTOWALTZ-3346 datatypemerge2 7138
Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions waltz-data/src/main/java/org/finos/waltz/data/data_type/DataTypeDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.finos.waltz.model.EntityLifecycleStatus;
import org.finos.waltz.model.EntityReference;
import org.finos.waltz.model.datatype.DataType;
import org.finos.waltz.model.datatype.DataTypeMigrationResult;
import org.finos.waltz.model.datatype.ImmutableDataType;
import org.finos.waltz.schema.tables.records.DataTypeRecord;
import org.jooq.Condition;
Expand Down Expand Up @@ -76,6 +77,12 @@ public DataTypeDao(DSLContext dsl) {
this.dsl = dsl;
}

public DataTypeMigrationResult migrate(Long fromId, Long toId, boolean deleteOldDataType) {
return dsl.transactionResult(ctx -> {
DSLContext tx = ctx.dsl();
return DataTypeUtilities.migrate(tx, fromId, toId, deleteOldDataType);
});
}

public List<DataType> findAll() {
return dsl
Expand Down
104 changes: 60 additions & 44 deletions ...s/waltz/jobs/tools/DataTypeUtilities.java → ...ltz/data/data_type/DataTypeUtilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package org.finos.waltz.jobs.tools;
package org.finos.waltz.data.data_type;

import org.finos.waltz.common.Checks;
import org.finos.waltz.common.SetUtilities;
import org.finos.waltz.model.EntityKind;
import org.finos.waltz.model.EntityLifecycleStatus;
import org.finos.waltz.model.EntityReference;
import org.finos.waltz.model.datatype.DataTypeMigrationResult;
import org.finos.waltz.model.datatype.ImmutableDataTypeMigrationResult;
import org.finos.waltz.schema.Tables;
import org.finos.waltz.schema.tables.DataType;
import org.finos.waltz.schema.tables.DataTypeUsage;
Expand All @@ -14,17 +17,14 @@
import org.finos.waltz.schema.tables.PhysicalSpecDataType;
import org.finos.waltz.schema.tables.PhysicalSpecification;
import org.finos.waltz.schema.tables.records.LogicalFlowRecord;
import org.finos.waltz.service.DIConfiguration;
import org.jooq.Condition;
import org.jooq.DSLContext;
import org.jooq.impl.DSL;
import org.jooq.lambda.tuple.Tuple;
import org.jooq.lambda.tuple.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
Expand All @@ -34,11 +34,12 @@
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import static org.finos.waltz.model.EntityReference.mkRef;
import static org.finos.waltz.schema.Tables.DATA_TYPE;

public class DataTypeUtilities {

private static final Logger LOG = LoggerFactory.getLogger(DataTypeUtilities.class);
private static final DataType dataType = Tables.DATA_TYPE;
private static final DataType dataType = DATA_TYPE;
private static final LogicalFlow logicalFlow = Tables.LOGICAL_FLOW;
private static final LogicalFlowDecorator logicalFlowDecorator = Tables.LOGICAL_FLOW_DECORATOR;
private static final FlowClassificationRule flowClassificationRule = Tables.FLOW_CLASSIFICATION_RULE;
Expand Down Expand Up @@ -106,6 +107,15 @@ private static long removeDataType(DSLContext dsl,
}


private static boolean removeDataType(DSLContext dsl,
long fromId) {
return dsl
.delete(dataType)
.where(dataType.ID.eq(fromId))
.execute() > 0;
}


public static void migrate(DSLContext dsl,
String fromCode,
String toCode,
Expand All @@ -132,20 +142,41 @@ public static void migrate(DSLContext dsl,
// 4) update flow_classification_rules set data_type_id = toId where data_type_id = fromId
// x) delete from dataType where code = 'fromCode'

migrateDataTypeUsage(dsl, fromId, toId);
migrateFlowClassificationRules(dsl, fromId, toId);
migrateLogicalFlowDecorator(dsl, fromId, toId);
migratePhysicalSpecDataType(dsl, fromId, toId);
migrate(dsl, fromId, toId, deleteOldDataType);

}

private static void verifyDataTypeHasNoChildren(DSLContext dsl, Long fromId) {
int childCount = dsl.fetchCount(DATA_TYPE, DATA_TYPE.PARENT_ID.eq(fromId));
Checks.checkTrue(childCount == 0, "Data Type, %d has %d children", fromId, childCount);
}

public static DataTypeMigrationResult migrate(DSLContext dsl, Long fromId, Long toId, boolean deleteOldDataType) {
if (deleteOldDataType) {
removeDataType(dsl, fromCode);
verifyDataTypeHasNoChildren(dsl, fromId);
}

int dtuCount = migrateDataTypeUsage(dsl, fromId, toId);
int crCount = migrateFlowClassificationRules(dsl, fromId, toId);
int lfCount = migrateLogicalFlowDecorator(dsl, fromId, toId);
int psCount = migratePhysicalSpecDataType(dsl, fromId, toId);
boolean dataTypeRemoved = deleteOldDataType
? removeDataType(dsl, fromId)
: false;

return ImmutableDataTypeMigrationResult
.builder()
.usageCount(dtuCount)
.classificationRuleCount(crCount)
.logicalFlowDataTypeCount(lfCount)
.physicalSpecDataTypeCount(psCount)
.dataTypeRemoved(dataTypeRemoved)
.build();
}

private static void migratePhysicalSpecDataType(DSLContext dsl,
Long fromId,
Long toId) {
private static int migratePhysicalSpecDataType(DSLContext dsl,
Long fromId,
Long toId) {
PhysicalSpecDataType physicSpec = physicalSpecDataType.as("physicSpec");
Condition notAlreadyExists = DSL
.notExists(DSL
Expand All @@ -167,12 +198,13 @@ private static void migratePhysicalSpecDataType(DSLContext dsl,
.execute();

LOG.info("Migrate Phys Spec Data Type Usage: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount);
return updateCount + rmCount;

}

private static void migrateLogicalFlowDecorator(DSLContext dsl,
Long fromId,
Long toId) {
private static int migrateLogicalFlowDecorator(DSLContext dsl,
Long fromId,
Long toId) {
LogicalFlowDecorator decorator = logicalFlowDecorator.as("decorator");

Condition notAlreadyExists = DSL.notExists(DSL
Expand All @@ -197,12 +229,14 @@ private static void migrateLogicalFlowDecorator(DSLContext dsl,
.execute();

LOG.info("Migrate Logical Flow Decorator: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount);

return updateCount + rmCount;
}


private static void migrateFlowClassificationRules(DSLContext dsl,
Long fromId,
Long toId) {
private static int migrateFlowClassificationRules(DSLContext dsl,
Long fromId,
Long toId) {

FlowClassificationRule authSrc = flowClassificationRule.as("authSrc");

Expand All @@ -228,12 +262,14 @@ private static void migrateFlowClassificationRules(DSLContext dsl,
.execute();

LOG.info("Migrate Flow Classification Rules: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount);
return updateCount + rmCount;

}


private static void migrateDataTypeUsage(DSLContext dsl,
Long fromId,
Long toId) {
private static int migrateDataTypeUsage(DSLContext dsl,
Long fromId,
Long toId) {
DataTypeUsage dtu = dataTypeUsage.as("dtu");

Condition condition = DSL.notExists(DSL
Expand All @@ -257,6 +293,8 @@ private static void migrateDataTypeUsage(DSLContext dsl,
.execute();

LOG.info("Migrate DataType Usage: {} -> {}, updated: {}, removed: {}", fromId, toId, updateCount, rmCount);

return updateCount + rmCount;
}

public static List<Long> findLogicalFlowIdsForDataType(DSLContext dsl, Long datatype, Set<Long> logicalFlowIds) {
Expand Down Expand Up @@ -366,26 +404,4 @@ public static long markDataTypeAsConcrete(DSLContext dsl,
.where(dataType.CODE.in(dataTypeCodes))
.execute();
}


public static void main(String[] args) throws IOException {

AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class);
DSLContext dsl = ctx.getBean(DSLContext.class);

String party = "DataTypeCode1";
String allocation = "DataTypeCode2";
String instrumentIdentifier = "DataTypeCode3";
String prospect = "DataTypeCode4";
String dealEvent = "DataTypeCode5";
String instrumentStatic = "DataTypeCode6";

dsl.transaction(context -> {
DSLContext tx = context.dsl();
migrate(tx, instrumentIdentifier, instrumentStatic, false);
migrate(tx, prospect, party, false);
migrate(tx, allocation, dealEvent, false);
//throw new RuntimeException("BoooM!");
});
}
}
14 changes: 14 additions & 0 deletions waltz-model/src/main/java/org/finos/waltz/model/datatype/DataTypeMigrationResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.finos.waltz.model.datatype;

import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.immutables.value.Value;

@Value.Immutable
@JsonSerialize(as=ImmutableDataTypeMigrationResult.class)
public interface DataTypeMigrationResult {
long usageCount();
long classificationRuleCount();
long logicalFlowDataTypeCount();
long physicalSpecDataTypeCount();
boolean dataTypeRemoved();
}
Loading
Loading