From 8fd117836f38b51e08e574af66187588e2b485da Mon Sep 17 00:00:00 2001 From: Jonathan Gamba Date: Tue, 21 Jan 2025 12:19:01 -0600 Subject: [PATCH] #30882 First commit for the ImportUtil refactor to improve responses. --- .../com/dotmarketing/util/ImportUtil.java | 1233 ++++++++++++++--- .../util/importer/AbstractContentSummary.java | 33 + .../util/importer/AbstractImportFileInfo.java | 33 + .../importer/AbstractImportHeaderInfo.java | 43 + .../AbstractImportHeaderValidationResult.java | 36 + .../util/importer/AbstractImportResult.java | 34 + .../importer/AbstractImportResultData.java | 28 + .../AbstractImportValidationMessage.java | 73 + .../util/importer/AbstractProcessedData.java | 28 + .../util/importer/HeaderValidationCodes.java | 37 + .../util/importer/ImportResultConverter.java | 175 +++ 11 files changed, 1523 insertions(+), 230 deletions(-) create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractContentSummary.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportFileInfo.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderInfo.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderValidationResult.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResult.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResultData.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportValidationMessage.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractProcessedData.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/HeaderValidationCodes.java create mode 100644 dotCMS/src/main/java/com/dotmarketing/util/importer/ImportResultConverter.java diff --git a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java index 9f9c41203d8..ea54d73c136 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java @@ -16,6 +16,7 @@ import com.dotmarketing.beans.Permission; import com.dotmarketing.business.APILocator; import com.dotmarketing.business.CacheLocator; +import com.dotmarketing.business.DotValidationException; import com.dotmarketing.business.PermissionAPI; import com.dotmarketing.cache.FieldsCache; import com.dotmarketing.common.model.ContentletSearch; @@ -49,6 +50,16 @@ import com.dotmarketing.portlets.structure.model.Structure; import com.dotmarketing.portlets.workflows.business.WorkflowAPI; import com.dotmarketing.portlets.workflows.model.WorkflowAction; +import com.dotmarketing.util.importer.AbstractImportValidationMessage.ValidationMessageType; +import com.dotmarketing.util.importer.ContentSummary; +import com.dotmarketing.util.importer.HeaderValidationCodes; +import com.dotmarketing.util.importer.ImportFileInfo; +import com.dotmarketing.util.importer.ImportHeaderInfo; +import com.dotmarketing.util.importer.ImportHeaderValidationResult; +import com.dotmarketing.util.importer.ImportResult; +import com.dotmarketing.util.importer.ImportResultData; +import com.dotmarketing.util.importer.ImportValidationMessage; +import com.dotmarketing.util.importer.ProcessedData; import com.liferay.portal.language.LanguageException; import com.liferay.portal.language.LanguageUtil; import com.liferay.portal.model.User; @@ -68,6 +79,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -280,9 +292,12 @@ public static HashMap> importFile(Long importId, String cur HashSet keyContentUpdated = new HashSet<>(); StringBuffer choosenKeyField = new StringBuffer(); + // Data structures to be populated by header validation HashMap headers = new HashMap<>(); HashMap keyFields = new HashMap<>(); HashMap relationships = new HashMap<>(); + HashMap onlyParent = new HashMap<>(); + HashMap onlyChild = new HashMap<>(); //Get unique fields for structure for(Field field : FieldsCache.getFieldsByStructureInode(contentType.getInode())){ @@ -294,15 +309,38 @@ public static HashMap> importFile(Long importId, String cur //Parsing the file line per line try { if ((csvHeaders != null) || (csvreader.readHeaders())) { - //Importing headers from the first file line - HashMap onlyParent=new HashMap<>(); - HashMap onlyChild=new HashMap<>(); + + // Process headers and get validation result + ImportHeaderValidationResult headerValidation; if (csvHeaders != null) { - importHeaders(csvHeaders, contentType, keyfields, isMultilingual, user, results, headers, keyFields, uniqueFields,relationships,onlyChild,onlyParent); + headerValidation = importHeaders(csvHeaders, contentType, keyfields, + isMultilingual, user, headers, keyFields, uniqueFields, relationships, + onlyChild, onlyParent); } else { - importHeaders(csvreader.getHeaders(), contentType, keyfields, isMultilingual, user, results, headers, keyFields, uniqueFields,relationships,onlyChild,onlyParent); + headerValidation = importHeaders(csvreader.getHeaders(), contentType, keyfields, + isMultilingual, user, headers, keyFields, uniqueFields, relationships, + onlyChild, onlyParent); } + + // Convert validation result to legacy format + for (ImportValidationMessage message : headerValidation.messages()) { + String messageText = message.message(); + switch (message.type()) { + case ERROR: + results.get("errors").add(messageText); + errors++; + break; + case WARNING: + results.get("warnings").add(messageText); + break; + case INFO: + results.get("messages").add(messageText); + break; + } + } + lineNumber++; + // Log preview/import status every 100 processed records //Reading the whole file if (headers.size() > 0) { @@ -429,281 +467,758 @@ public static HashMap> importFile(Long importId, String cur return results; } - /** - * Reads the CSV file headers in order to find inconsistencies or errors. - * Such situations will be saved in the {@code results} list. - * - * @param headerLine - * - The line in the CSV file containing the data headers. - * @param contentType - * - The Content Type that the data in this file is associated - * to. - * @param keyFieldsInodes - * - The Inodes of the fields used to associated existing dotCMS - * contentlets with the information in this file. Can be empty. - * @param isMultilingual - * - If set to {@code true}, the CSV file will import contents in - * more than one language. Otherwise, set to {@code false}. - * @param user - * - The {@link User} performing this action. - * @param results - * - The status object that keeps track of potential errors, - * inconsistencies, or warnings. - * @param headers - * @param keyFields - * - The fields used to associated existing dotCMS contentlets - * with the information in this file. Can be empty. - * @param uniqueFields - * - The list of fields that are unique (if any). - * @param relationships - * - Content relationships (if any). - * @param onlyChild - * - Contains content relationships that are only child - * relationships (header name ends with {@code "-RELCHILD"}). - * @param onlyParent - * - Contains content relationships that are only parent - * relationships (header name ends with {@code "-RELPARENT"}). - * @throws Exception - * An error occurred when validating the CSV data. - */ - private static void importHeaders(String[] headerLine, Structure contentType, String[] keyFieldsInodes, boolean isMultilingual, User user, HashMap> results, HashMap headers, HashMap keyFields, List uniqueFields, HashMap relationships,HashMap onlyChild, HashMap onlyParent) throws Exception { - - int importableFields = 0; - - //Importing headers and storing them in a hashmap to be reused later in the whole import process - final List fields = FieldsCache.getFieldsByStructureInode(contentType.getInode()); - final List contentTypeRelationships = APILocator.getRelationshipAPI() - .byContentType(contentType); - final Map contentTypeRelationshipsMap = new LowerKeyMap<>(); - final List requiredFields = new ArrayList<>(); + /** + * Validates and processes the headers from a CSV import file. This method performs + * comprehensive validation of the header line including: + *
    + *
  • Basic format validation
    • + *
  • Content type field matching
    • + *
  • Relationship field processing
    • + *
  • Multilingual requirements
    • + *
  • Key fields validation
    • + *
  • Unique fields processing
    • + *
+ * + * @param headerLine CSV file header line to validate + * @param contentType Content Type structure to validate against + * @param keyFieldsInodes Array of field inodes used as keys for content matching + * @param isMultilingual Whether the import supports multiple languages + * @param user User performing the import + * @param headers Map to store validated header-to-field mappings + * @param keyFields Map to store validated key field mappings + * @param uniqueFields List of fields marked as unique in the content type + * @param relationships Map to store relationship field mappings + * @param onlyChild Map tracking child-only relationships + * @param onlyParent Map tracking parent-only relationships + * @return Validation result containing header information and validation messages + * @throws Exception if validation fails or processing encounters errors + */ + private static ImportHeaderValidationResult importHeaders(String[] headerLine, + Structure contentType, String[] keyFieldsInodes, boolean isMultilingual, User user, + HashMap headers, HashMap keyFields, + List uniqueFields, HashMap relationships, + HashMap onlyChild, HashMap onlyParent) + throws Exception { + + // Create structured results for validation tracking + final var validationBuilder = ImportHeaderValidationResult.builder(); final List headerFields = new ArrayList<>(); - //Saves relationships as map for efficient search in getRelationships method - contentTypeRelationshipsMap.putAll(contentTypeRelationships.stream().collect( - Collectors.toMap(Relationship::getRelationTypeValue, Function.identity()))); + // Validate basic header format + validateHeaderLineFormat(headerLine, user, validationBuilder); + + // Get content type info and create relationship map + final ContentTypeInfo typeInfo = getContentTypeInfo(contentType); + final Map relationshipsMap = createRelationshipMap( + typeInfo.relationships); + + // Process and validate headers + processHeaders( + headerLine, contentType, keyFieldsInodes, isMultilingual, + relationshipsMap, headerFields, headers, keyFields, + relationships, onlyChild, onlyParent, user, validationBuilder); + + // Validate multilingual requirements if needed + validateMultilingualHeaders(isMultilingual, headerFields, validationBuilder); + + // Validate key fields and unique fields + validateKeyFields(keyFieldsInodes, headers, user, validationBuilder); + processUniqueFields(uniqueFields, user, validationBuilder); + + // Generate summary messages + addSummaryMessages(headers.size(), typeInfo.importableFields, + relationships.size(), user, validationBuilder); + + // Add context information + Map context = new HashMap<>(); + context.put("headers", headers); + context.put("keyFields", keyFields); + context.put("relationships", relationships); + context.put("onlyChild", onlyChild); + context.put("onlyParent", onlyParent); + validationBuilder.context(context); + + return validationBuilder.build(); + } - for(Field field:fields){ - if(field.isRequired()){ - requiredFields.add(field.getVelocityVarName()); - } - } + /** + * Processes and validates header entries from the CSV file. This method handles the detailed + * validation of each header column and populates various data structures with the results. + * + * @param headerLine Array of header strings to process + * @param contentType Content Type structure to validate against + * @param keyFieldsInodes Array of field inodes used as keys + * @param isMultilingual Whether import is multilingual + * @param relationshipsMap Map of available relationships + * @param headerFields List to store processed header names + * @param headers Map to store header-to-field mappings + * @param keyFields Map to store key field mappings + * @param relationships Map to store relationship mappings + * @param onlyChild Map for child-only relationships + * @param onlyParent Map for parent-only relationships + * @param user User performing the import + * @param validationBuilder Builder for validation result + * @throws Exception if processing encounters errors + */ + private static void processHeaders(String[] headerLine, + Structure contentType, + String[] keyFieldsInodes, boolean isMultilingual, + Map relationshipsMap, List headerFields, + HashMap headers, HashMap keyFields, + HashMap relationships, + HashMap onlyChild, HashMap onlyParent, + User user, ImportHeaderValidationResult.Builder validationBuilder) throws Exception { + + List validHeaders = new ArrayList<>(); + List invalidHeaders = new ArrayList<>(); + + // Process each header for (int i = 0; i < headerLine.length; i++) { - boolean found = false; String header = headerLine[i].replaceAll("'", ""); + headerFields.add(header); - if (header.equalsIgnoreCase("Identifier")) { - results.get("messages").add(LanguageUtil.get(user, "identifier-field-found-in-import-contentlet-csv-file")); - results.get("identifiers").add("" + i); - continue; - } - if (header.equalsIgnoreCase(Contentlet.WORKFLOW_ACTION_KEY)) { - results.get("messages").add(LanguageUtil.get(user, "workflow-action-id-field-found-in-import-contentlet-csv-file")); - results.get(Contentlet.WORKFLOW_ACTION_KEY).add(StringPool.BLANK + i); + // Handle special headers first + if (isSpecialHeader(header, i, user, validationBuilder)) { + validHeaders.add(header); continue; } - headerFields.add(header); + // Process and validate header + processAndValidateHeader( + header, i, contentType, headers, keyFieldsInodes, keyFields, + relationships, onlyChild, onlyParent, relationshipsMap, + validHeaders, invalidHeaders, isMultilingual, user, validationBuilder); + } - for (Field field : fields) { - if (field.getVelocityVarName().equalsIgnoreCase(header)) { - if (field.getFieldType().equals(Field.FieldType.BUTTON.toString())){ - found = true; - results.get("warnings").add( - LanguageUtil.get(user, "Header")+" \"" + header + // Validate required fields + List missingHeaders = validateRequiredFields(headerFields, user, contentType, + validationBuilder); + + // Create headerInfo + final var headerInfo = ImportHeaderInfo.builder() + .totalHeaders(headerLine.length) + .validHeaders(validHeaders.toArray(new String[0])) + .invalidHeaders(invalidHeaders.toArray(new String[0])) + .missingHeaders(missingHeaders.toArray(new String[0])) + .validationDetails(new HashMap<>()) // Add validation details if needed + .build(); + validationBuilder.headerInfo(headerInfo); + } - +"\" "+ LanguageUtil.get(user, "matches-a-field-of-type-button-this-column-of-data-will-be-ignored")); - } - else if (field.getFieldType().equals(Field.FieldType.LINE_DIVIDER.toString())){ - found = true; - results.get("warnings").add( - LanguageUtil.get(user, "Header")+" \"" + header - + "\" "+LanguageUtil.get(user, "matches-a-field-of-type-line-divider-this-column-of-data-will-be-ignored")); - } - else if (field.getFieldType().equals(Field.FieldType.TAB_DIVIDER.toString())){ - found = true; - results.get("warnings").add( - LanguageUtil.get(user, "Header")+" \"" + header - + "\" "+LanguageUtil.get(user, "matches-a-field-of-type-tab-divider-this-column-of-data-will-be-ignored")); - } - else { - found = true; - headers.put(i, field); - for (String fieldInode : keyFieldsInodes) { - if (fieldInode.equals(field.getInode())) - keyFields.put(i, field); - } + /** + * Validates the basic format of header lines from the CSV file. Checks for: + *
    + *
  • Non-empty header line
    • + *
  • No duplicate header names
    • + *
+ * + * @param headerLine Array of header strings to validate + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @throws LanguageException If language key lookup fails + * @throws DotValidationException If headers are invalid or empty + */ + private static void validateHeaderLineFormat(String[] headerLine, User user, + ImportHeaderValidationResult.Builder validationBuilder) throws LanguageException { + + if (headerLine == null || headerLine.length == 0) { + validationBuilder.addMessages(ImportValidationMessage.builder() + .type(ValidationMessageType.ERROR) + .code(HeaderValidationCodes.INVALID_HEADER_FORMAT.name()) + .message(LanguageUtil.get(user, + "No-headers-found-on-the-file-nothing-will-be-imported")) + .build()); + throw new DotValidationException("Invalid header format"); + } - if (field.getFieldType().equals(FieldType.RELATIONSHIP.toString())) { - - final Relationship fieldRelationship = APILocator.getRelationshipAPI() - .getRelationshipFromField(field, user); - contentTypeRelationshipsMap - .put(field.getVelocityVarName().toLowerCase(), - fieldRelationship); - contentTypeRelationshipsMap - .remove(fieldRelationship.getRelationTypeValue().toLowerCase()); - - //considering case when importing self-related content - if (fieldRelationship.getChildStructureInode() - .equals(fieldRelationship.getParentStructureInode())) { - if (fieldRelationship.getParentRelationName() != null - && fieldRelationship.getParentRelationName() - .equals(field.getVelocityVarName())) { - onlyParent.put(i, true); - onlyChild.put(i, false); - } else if (fieldRelationship.getChildRelationName() != null - && fieldRelationship.getChildRelationName() - .equals(field.getVelocityVarName())) { - onlyParent.put(i, false); - onlyChild.put(i, true); - } - } - } - break; - } - } + // Validate no duplicate headers + Set uniqueHeaders = new HashSet<>(); + for (int i = 0; i < headerLine.length; i++) { + String header = headerLine[i].replaceAll("'", "").toLowerCase(); + if (!uniqueHeaders.add(header)) { + validationBuilder.addMessages(ImportValidationMessage.builder() + .type(ValidationMessageType.ERROR) + .code(HeaderValidationCodes.DUPLICATE_HEADER.name()) + .field(headerLine[i]) + .lineNumber(1) + .message(LanguageUtil.get(user, "Duplicate-header-found") + ": " + + headerLine[i]) + .build()); } + } + } - /* - * We gonna delete -RELPARENT -RELCHILD so we can - * search for the relation name. No problem as - * we put relationships.put(i,relationship) instead - * of header. - */ - boolean onlyP=false; - if(header.endsWith("-RELPARENT")) { - header = header.substring(0,header.lastIndexOf("-RELPARENT")); - onlyP=true; - } + /** + * Processes and validates an individual header entry. This method attempts to match the header + * with content type fields or relationships and records the validation result. + * + * @param header Header string to process + * @param columnIndex Index of the header in the CSV file + * @param contentType Content Type structure to validate against + * @param headers Map to populate with validated header-to-field mappings + * @param keyFieldsInodes Array of field inodes used as keys + * @param keyFields Map to populate with key field mappings + * @param relationships Map to populate with relationship mappings + * @param onlyChild Map tracking child-only relationships + * @param onlyParent Map tracking parent-only relationships + * @param relationshipsMap Map of available relationships + * @param validHeaders List to store valid header names + * @param invalidHeaders List to store invalid header names + * @param isMultilingual Whether import supports multiple languages + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @throws LanguageException If language key lookup fails + */ + private static void processAndValidateHeader( + String header, int columnIndex, Structure contentType, + HashMap headers, String[] keyFieldsInodes, + HashMap keyFields, + HashMap relationships, + HashMap onlyChild, + HashMap onlyParent, + Map relationshipsMap, + List validHeaders, List invalidHeaders, + boolean isMultilingual, User user, + ImportHeaderValidationResult.Builder validationBuilder) + throws LanguageException { - boolean onlyCh=false; - if(header.endsWith("-RELCHILD")) { - header = header.substring(0,header.lastIndexOf("-RELCHILD")); - onlyCh=true; - } + // First try content type fields + boolean found = processContentTypeField( + header, columnIndex, contentType, headers, + keyFieldsInodes, keyFields, relationships, + onlyChild, onlyParent, relationshipsMap, user, validationBuilder); - found = getRelationships(relationships, onlyChild, onlyParent, i, found, - header, onlyP, onlyCh, contentTypeRelationshipsMap); + if (found) { + validHeaders.add(header); + return; + } - if ((!found) && !(isMultilingual && (header.equals(languageCodeHeader) || header.equals(countryCodeHeader)))) { - results.get("warnings").add( - LanguageUtil.get(user, "Header")+" \"" + header - + "\""+ " "+ LanguageUtil.get(user, "doesn-t-match-any-structure-field-this-column-of-data-will-be-ignored")); - } + // Then try relationships + found = processRelationshipField( + header, columnIndex, relationshipsMap, + relationships, onlyChild, onlyParent); + + if (found) { + validHeaders.add(header); + return; } - requiredFields.removeAll(headerFields); + // If not found and not a language header, mark as invalid + if (!(isMultilingual && isLanguageHeader(header))) { + invalidHeaders.add(header); + validationBuilder.addMessages(ImportValidationMessage.builder() + .type(ValidationMessageType.WARNING) + .code(HeaderValidationCodes.INVALID_HEADER.name()) + .field(header) + .lineNumber(1) + .context(Map.of("columnIndex", columnIndex)) + .message(LanguageUtil.get(user, "Header") + " \"" + header + "\" " + + LanguageUtil.get(user, + "doesn-t-match-any-structure-field-this-column-of-data-will-be-ignored")) + .build()); + } + } - for(String requiredField: requiredFields){ - results.get("errors").add(LanguageUtil.get(user, "Field")+": \"" + requiredField+ "\" "+LanguageUtil.get(user, "required-field-not-found-in-header")); + /** + * Validates headers required for multilingual imports. Checks for presence of language code and + * country code headers when multilingual import is enabled. + * + * @param isMultilingual Whether multilingual import is enabled + * @param headerFields List of processed header fields + * @param validationBuilder Builder to accumulate validation messages + */ + private static void validateMultilingualHeaders(boolean isMultilingual, + List headerFields, ImportHeaderValidationResult.Builder validationBuilder) { + + if (!isMultilingual) { + return; + } + + boolean hasLanguageCode = headerFields.contains(languageCodeHeader); + boolean hasCountryCode = headerFields.contains(countryCodeHeader); + + if (!hasLanguageCode || !hasCountryCode) { + validationBuilder.addMessages(ImportValidationMessage.builder() + .type(ValidationMessageType.ERROR) + .code(HeaderValidationCodes.INVALID_LANGUAGE.name()) + .message("languageCode and countryCode fields are mandatory in the CSV file" + + " when importing multilingual content") + .lineNumber(1) + .build()); } + } + + /** + * Retrieves and processes Content Type information including fields and relationships. + * + * @param contentType Content Type structure to process + * @return ContentTypeInfo containing fields, relationships and count of importable fields + */ + private static ContentTypeInfo getContentTypeInfo(Structure contentType) { + final List fields = FieldsCache.getFieldsByStructureInode(contentType.getInode()); + final List relationships = APILocator.getRelationshipAPI() + .byContentType(contentType); + int importableFields = 0; for (Field field : fields) { - if (isImportableField(field)){ + if (isImportableField(field)) { importableFields++; } } - //Checking keyField selected by the user against the headers - for (String keyField : keyFieldsInodes) { - boolean found = false; - for (Field headerField : headers.values()) { - if (headerField.getInode().equals(keyField)) { - found = true; - break; + return new ContentTypeInfo(fields, relationships, importableFields); + } + + /** + * Creates a map of relationships for efficient lookup during header validation. Uses the + * relationship type value as the key for quick access. + * + * @param relationships List of relationships to map + * @return Map of relationships keyed by their type value + */ + private static Map createRelationshipMap( + List relationships) { + final Map relationshipsMap = new LowerKeyMap<>(); + relationshipsMap.putAll(relationships.stream() + .collect( + Collectors.toMap( + Relationship::getRelationTypeValue, + Function.identity() + ))); + return relationshipsMap; + } + + /** + * Collects all required fields from a list of content type fields. + * + * @param fields List of fields to process + * @return List of velocity variable names for required fields + */ + private static List collectRequiredFields(List fields) { + return fields.stream() + .filter(Field::isRequired) + .map(Field::getVelocityVarName) + .collect(Collectors.toList()); + } + + /** + * Checks if a header represents a special system field like Identifier or Workflow Action. Adds + * appropriate validation messages for recognized system headers. + * + * @param header Header to check + * @param columnIndex Index of the header in the CSV file + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @return true if header is a special system header, false otherwise + * @throws LanguageException If language key lookup fails + */ + private static boolean isSpecialHeader(String header, int columnIndex, User user, + ImportHeaderValidationResult.Builder validationBuilder) throws LanguageException { + + if (header.equalsIgnoreCase("Identifier")) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .code(HeaderValidationCodes.SYSTEM_HEADER.name()) + .field("Identifier") + .lineNumber(1) + .message(LanguageUtil.get(user, + "identifier-field-found-in-import-contentlet-csv-file")) + .context(Map.of("columnIndex", columnIndex)) + .build() + ); + return true; + } + + if (header.equalsIgnoreCase(Contentlet.WORKFLOW_ACTION_KEY)) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .code(HeaderValidationCodes.SYSTEM_HEADER.name()) + .field(Contentlet.WORKFLOW_ACTION_KEY) + .lineNumber(1) + .message(LanguageUtil.get(user, + "workflow-action-id-field-found-in-import-contentlet-csv-file")) + .context(Map.of("columnIndex", columnIndex)) + .build() + ); + return true; + } + + return false; + } + + /** + * Processes and validates a header against content type fields. Handles field mapping, key + * field identification, and relationship setup. + * + * @param header Header to process + * @param columnIndex Index of the header in the CSV file + * @param contentType Content Type structure to validate against + * @param headers Map to populate with header-to-field mappings + * @param keyFieldsInodes Array of field inodes used as keys + * @param keyFields Map to populate with key field mappings + * @param relationships Map to populate with relationship mappings + * @param onlyChild Map tracking child-only relationships + * @param onlyParent Map tracking parent-only relationships + * @param relationshipsMap Map of available relationships + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @return true if header was successfully processed as a content type field + * @throws LanguageException If language key lookup fails + */ + private static boolean processContentTypeField(String header, int columnIndex, + Structure contentType, HashMap headers, + String[] keyFieldsInodes, HashMap keyFields, + HashMap relationships, + HashMap onlyChild, HashMap onlyParent, + Map relationshipsMap, User user, + ImportHeaderValidationResult.Builder validationBuilder) throws LanguageException { + + for (Field field : FieldsCache.getFieldsByStructureInode(contentType.getInode())) { + if (!field.getVelocityVarName().equalsIgnoreCase(header)) { + continue; + } + + if (isNonImportableField(field)) { + validationBuilder.addMessages(ImportValidationMessage.builder() + .type(ValidationMessageType.WARNING) + .code(HeaderValidationCodes.INVALID_HEADER.name()) + .field(header) + .lineNumber(1) + .message(formatNonImportableFieldMessage(field, header, user)) + .context(Map.of("columnIndex", columnIndex)) + .build()); + return true; + } + + // Add field to headers + headers.put(columnIndex, field); + + // Check if field matches any key field inodes and add to keyFields if it does + for (String fieldInode : keyFieldsInodes) { + if (fieldInode.equals(field.getInode())) { + keyFields.put(columnIndex, field); } } - if (!found) { - results.get("errors").add( - LanguageUtil.get(user, "Key-field")+": \"" + FieldFactory.getFieldByInode(keyField).getVelocityVarName() - + "\" "+LanguageUtil.get(user, "choosen-doesn-t-match-any-of-theh-eaders-found-in-the-file")); + + // Add relationships if field is relationship type + if (field.getFieldType().equals(FieldType.RELATIONSHIP.toString())) { + processRelationshipField(header, columnIndex, relationshipsMap, relationships, + onlyChild, onlyParent); } + + return true; } - if (keyFieldsInodes.length == 0) { - Logger.debug(ImportUtil.class, ()->"Warning: No-key-fields-were-choosen-it-could-give-to-you-duplicated-content"); - results.get("warnings").add( - LanguageUtil.get(user, - "No-key-fields-were-choosen-it-could-give-to-you-duplicated-content")); + return false; + } + + /** + * Checks if a field is non-importable based on its type. Non-importable fields include buttons, + * line dividers, and tab dividers. + * + * @param field Field to check + * @return true if field is non-importable, false otherwise + */ + private static boolean isNonImportableField(Field field) { + return field.getFieldType().equals(Field.FieldType.BUTTON.toString()) || + field.getFieldType().equals(Field.FieldType.LINE_DIVIDER.toString()) || + field.getFieldType().equals(Field.FieldType.TAB_DIVIDER.toString()); + } + + /** + * Formats an error message for non-importable field types. + * + * @param field Field that is non-importable + * @param header Header name from CSV + * @param user User performing the import + * @return Formatted error message + * @throws LanguageException If language key lookup fails + */ + private static String formatNonImportableFieldMessage(Field field, String header, + User user) throws LanguageException { + return LanguageUtil.get(user, "Header") + " \"" + header + "\" " + + LanguageUtil.get(user, "matches-a-field-of-type-" + + field.getFieldType().toLowerCase() + + "-this-column-of-data-will-be-ignored"); + } + + /** + * Processes a header as a potential relationship field. Handles both parent and child + * relationship designations through suffix parsing (-RELPARENT/-RELCHILD). + * + * @param header Header to process + * @param columnIndex Index of the header in the CSV file + * @param relationshipsMap Map of available relationships + * @param relationships Map to populate with relationship mappings + * @param onlyChild Map tracking child-only relationships + * @param onlyParent Map tracking parent-only relationships + * @return true if header was successfully processed as a relationship field + */ + private static boolean processRelationshipField(String header, int columnIndex, + Map relationshipsMap, + HashMap relationships, + HashMap onlyChild, HashMap onlyParent) { + + String relationshipHeader = header; + boolean onlyP = false; + boolean onlyCh = false; + + if (header.endsWith("-RELPARENT")) { + relationshipHeader = header.substring(0, header.lastIndexOf("-RELPARENT")); + onlyP = true; + } else if (header.endsWith("-RELCHILD")) { + relationshipHeader = header.substring(0, header.lastIndexOf("-RELCHILD")); + onlyCh = true; + } + + final Relationship relationship = relationshipsMap.get(relationshipHeader.toLowerCase()); + if (relationship == null) { + return false; } - if(!uniqueFields.isEmpty()){ - for(Field f : uniqueFields){ + relationships.put(columnIndex, relationship); - Logger.debug(ImportUtil.class, ()->"the-structure-field" + " " + f.getVelocityVarName() + " " + "is-unique"); - results.get("warnings").add(LanguageUtil.get(user, "the-structure-field")+ " " + f.getVelocityVarName() + " " +LanguageUtil.get(user, "is-unique")); - } + if (!onlyParent.containsKey(columnIndex)) { + onlyParent.put(columnIndex, onlyP); } - //Adding some messages to the results - if (importableFields == headers.size()) { - results.get("messages").add( - LanguageUtil.get(user, headers.size() + " "+LanguageUtil.get(user, "headers-match-these-will-be-imported"))); + if (!onlyChild.containsKey(columnIndex)) { + setSelfRelationshipType(relationship, onlyCh, columnIndex, onlyChild); + } + + return true; + } + + /** + * Sets up relationship type for self-referencing relationships. Handles special case where + * parent and child structures are the same. + * + * @param relationship The relationship being processed + * @param onlyCh Flag indicating if header is explicitly marked as child + * @param columnIndex Index of the header in the CSV file + * @param onlyChild Map to update with relationship type + */ + private static void setSelfRelationshipType(Relationship relationship, + boolean onlyCh, int columnIndex, HashMap onlyChild) { + if (relationship.getChildStructureInode().equals(relationship.getParentStructureInode()) + && !onlyCh) { + onlyChild.put(columnIndex, true); } else { - if (headers.size() > 0) { - results.get("messages").add(headers.size() + " " + LanguageUtil.get(user, "headers-found-on-the-file-matches-all-the-structure-fields")); - } else { - results - .get("messages") - .add( - LanguageUtil.get(user, "No-headers-found-on-the-file-that-match-any-of-the-structure-fields")); + onlyChild.put(columnIndex, onlyCh); + } + } + + /** + * Checks if a header is one of the special language-related fields. + * + * @param header Header to check + * @return true if header is a language or country code field + */ + private static boolean isLanguageHeader(String header) { + return header.equals(languageCodeHeader) || header.equals(countryCodeHeader); + } + + /** + * Validates required fields against the headers found in the CSV. Identifies missing required + * fields and adds appropriate error messages. + * + * @param headerFields List of headers found in CSV + * @param user User performing the import + * @param contentType Content Type structure to validate against + * @param validationBuilder Builder to accumulate validation messages + * @return List of required fields that were not found in headers + * @throws LanguageException If language key lookup fails + */ + private static List validateRequiredFields(List headerFields, User user, + Structure contentType, ImportHeaderValidationResult.Builder validationBuilder) + throws LanguageException { + + // Get required fields + List requiredFields = collectRequiredFields( + FieldsCache.getFieldsByStructureInode(contentType.getInode())); + + // Find missing required fields + List missingRequired = new ArrayList<>(requiredFields); + missingRequired.removeAll(headerFields); + + // Add errors for missing required fields + for (String requiredField : missingRequired) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.ERROR) + .code(HeaderValidationCodes.REQUIRED_FIELD_MISSING.name()) + .field(requiredField) + .lineNumber(1) + .message( + LanguageUtil.get(user, "Field") + + ": \"" + requiredField + "\" " + + LanguageUtil.get(user, + "required-field-not-found-in-header")) + .build() + ); + } + + return missingRequired; + } + + /** + * Validates key fields specified for the import. Ensures all specified key fields are present + * in the headers. + * + * @param keyFieldsInodes Array of field inodes used as keys + * @param headers Map of validated header-to-field mappings + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @throws LanguageException If language key lookup fails + */ + private static void validateKeyFields(String[] keyFieldsInodes, + HashMap headers, User user, + ImportHeaderValidationResult.Builder validationBuilder) throws LanguageException { + + if (keyFieldsInodes.length == 0) { + + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.WARNING) + .code(HeaderValidationCodes.NO_KEY_FIELDS.name()) + .lineNumber(1) + .message(LanguageUtil.get(user, + "No-key-fields-were-choosen-it-could-give-to-you-duplicated-content")) + .build() + ); + return; + } + + for (String keyFieldInode : keyFieldsInodes) { + boolean found = false; + for (Field headerField : headers.values()) { + if (headerField.getInode().equals(keyFieldInode)) { + found = true; + break; + } + } + + if (!found) { + Field keyField = FieldFactory.getFieldByInode(keyFieldInode); + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.ERROR) + .code(HeaderValidationCodes.INVALID_KEY_FIELD.name()) + .field(keyField.getVelocityVarName()) + .lineNumber(1) + .message(LanguageUtil.get(user, "Key-field") + ": \"" + + keyField.getVelocityVarName() + "\" " + + LanguageUtil.get(user, + "choosen-doesn-t-match-any-of-theh-eaders-found-in-the-file")) + .build() + ); } + } + } - Logger.debug(ImportUtil.class, ()->"Not-all-the-structure-fields-were-matched-against-the-file-headers-Some-content-fields-could-be-left-empty"); - results - .get("warnings") - .add(LanguageUtil.get(user, "Not-all-the-structure-fields-were-matched-against-the-file-headers-Some-content-fields-could-be-left-empty")); + /** + * Processes unique fields and adds appropriate warning messages. Alerts users about fields that + * require unique values. + * + * @param uniqueFields List of fields marked as unique + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @throws LanguageException If language key lookup fails + */ + private static void processUniqueFields(List uniqueFields, User user, + ImportHeaderValidationResult.Builder validationBuilder) throws LanguageException { + if (uniqueFields.isEmpty()) { + return; } - //Adding the relationship messages - if(relationships.size() > 0) - { - results.get("messages").add(LanguageUtil.get(user, relationships.size() + " "+LanguageUtil.get(user, "relationship-match-these-will-be-imported"))); + + for (Field uniqueField : uniqueFields) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.WARNING) + .code(HeaderValidationCodes.UNIQUE_FIELD.name()) + .field(uniqueField.getVelocityVarName()) + .message(LanguageUtil.get(user, "the-structure-field") + " " + + uniqueField.getVelocityVarName() + " " + + LanguageUtil.get(user, "is-unique")) + .build() + ); } } /** + * Adds summary messages about the header validation process. Provides information about: + *
    + *
  • Number of matched headers
    • + *
  • Completeness of field coverage
    • + *
  • Relationship matches
    • + *
* - * @param relationships - * @param onlyChild - * @param onlyParent - * @param i - * @param found - * @param header - * @param onlyP - * @param onlyCh - * @param contentTypeRelationshipsMap - * @return + * @param headerCount Number of valid headers found + * @param importableFieldCount Number of fields that can be imported + * @param relationshipCount Number of relationships found + * @param user User performing the import + * @param validationBuilder Builder to accumulate validation messages + * @throws LanguageException If language key lookup fails */ - private static boolean getRelationships( - final HashMap relationships, - final HashMap onlyChild, - final HashMap onlyParent, final int i, boolean found, - final String header, - final boolean onlyP, final boolean onlyCh, - final Map contentTypeRelationshipsMap) { - - final Relationship relationship = contentTypeRelationshipsMap.get(header.toLowerCase()); - - //Check if the header is a relationship - if (relationship != null) { - found = true; - relationships.put(i, relationship); - - if (!onlyParent.containsKey(i)){ - onlyParent.put(i, onlyP); - } + private static void addSummaryMessages(int headerCount, int importableFieldCount, + int relationshipCount, User user, + ImportHeaderValidationResult.Builder validationBuilder) + throws LanguageException { - if (!onlyChild.containsKey(i)) { - // special case when the relationship has the same structure for parent and child, set only as child - if (relationship.getChildStructureInode().equals(relationship.getParentStructureInode()) - && !onlyCh && !onlyP) { - onlyChild.put(i, true); - }else{ - onlyChild.put(i, onlyCh); - } + if (headerCount == importableFieldCount) { + + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .message(LanguageUtil.get(user, headerCount + " " + + LanguageUtil.get(user, "headers-match-these-will-be-imported"))) + .build() + ); + } else { + if (headerCount > 0) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .message(headerCount + " " + + LanguageUtil.get(user, + "headers-found-on-the-file-matches-all-the-structure-fields")) + .build() + ); + } else { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .message(LanguageUtil.get(user, + "No-headers-found-on-the-file-that-match-any-of-the-structure-fields")) + .build() + ); } + + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.WARNING) + .code(HeaderValidationCodes.INCOMPLETE_HEADERS.name()) + .message(LanguageUtil.get(user, + "Not-all-the-structure-fields-were-matched-against-the-file-headers" + + + "-Some-content-fields-could-be-left-empty")) + .lineNumber(1) + .build() + ); + } + + if (relationshipCount > 0) { + validationBuilder.addMessages( + ImportValidationMessage.builder() + .type(ValidationMessageType.INFO) + .message(LanguageUtil.get(user, relationshipCount + " " + + LanguageUtil.get(user, + "relationship-match-these-will-be-imported"))) + .build() + ); } - return found; } /** @@ -2431,4 +2946,262 @@ private static String getErrorMsgFromException(final User user, final DotRuntime } } + /** + * Container class holding content type information needed for header validation. This includes + * fields, relationships, and a count of importable fields. + */ + private static class ContentTypeInfo { + + /** + * List of fields defined in the content type + */ + final List fields; + + /** + * List of relationships associated with the content type + */ + final List relationships; + + /** + * Count of fields that can be imported + */ + final int importableFields; + + /** + * Creates a new ContentTypeInfo instance. + * + * @param fields List of fields in the content type + * @param relationships List of relationships for the content type + * @param importableFields Count of fields that can be imported + */ + ContentTypeInfo(final List fields, final List relationships, + final int importableFields) { + this.fields = fields; + this.relationships = relationships; + this.importableFields = importableFields; + } + } + + /** + * Container class for managing structured import results during the validation process. This + * class handles building and updating the structured result format. + */ + private static class ImportResults { + + /** + * Builder for the structured results + */ + private final ImportResult.Builder structuredResults; + + /** + * Creates a new ImportResults instance with initialized data structures. + */ + ImportResults() { + this.structuredResults = initializeStructuredResults(); + } + + /** + * Initializes the base structure for import results. + * + * @return Builder configured with default values + */ + private static ImportResult.Builder initializeStructuredResults() { + return ImportResult.builder() + .fileInfo(ImportFileInfo.builder() + .totalRows(0) + .parsedRows(0) + .headerInfo(initializeHeaderInfo()) + .build()) + .data(initializeResultData()); + } + + /** + * Initializes header information with empty arrays. + * + * @return HeaderInfo builder with default values + */ + private static ImportHeaderInfo initializeHeaderInfo() { + return ImportHeaderInfo.builder() + .validHeaders(new String[0]) + .invalidHeaders(new String[0]) + .missingHeaders(new String[0]) + .validationDetails(new HashMap<>()) + .build(); + } + + /** + * Initializes result data with zero counters. + * + * @return ResultData builder with default values + */ + private static ImportResultData initializeResultData() { + return ImportResultData.builder() + .processed(ProcessedData.builder() + .valid(0) + .invalid(0) + .build()) + .summary(ContentSummary.builder() + .created(0) + .updated(0) + .contentType("") + .build()) + .build(); + } + + /** + * Adds multiple validation messages to the results. + * + * @param validationMessages List of messages to add + */ + void addMessages(List validationMessages) { + if (validationMessages != null) { + validationMessages.forEach(structuredResults::addMessages); + } + } + + /** + * Adds a single validation message to the results. + * + * @param message Message to add + */ + void addMessage(ImportValidationMessage message) { + structuredResults.addMessages(message); + } + + /** + * Updates file processing information. + * + * @param fileInfo Updated file information + */ + void updateFileInfo(final ImportFileInfo fileInfo) { + structuredResults.fileInfo(fileInfo); + } + + /** + * Updates header validation information. + * + * @param headerInfo Updated header information + */ + void updateHeaderInfo(final ImportHeaderInfo headerInfo) { + ImportFileInfo currentFileInfo = structuredResults.build().fileInfo(); + structuredResults.fileInfo(currentFileInfo.withHeaderInfo(headerInfo)); + } + + /** + * Updates the content type name in the results. + * + * @param contentType Name of the content type + */ + void updateContentType(String contentType) { + ImportResultData currentData = structuredResults.build().data(); + structuredResults.data(currentData.withSummary( + currentData.summary().withContentType(contentType) + )); + } + + /** + * Updates processing counters in the results. + * + * @param counts Updated counter values + */ + void updateCounters(ImportCounts counts) { + structuredResults.data(ImportResultData.builder() + .processed(ProcessedData.builder() + .valid(counts.getValid()) + .invalid(counts.getInvalid()) + .build()) + .summary(ContentSummary.builder() + .created(counts.getCreated()) + .updated(counts.getUpdated()) + .contentType(structuredResults.build().data().summary().contentType()) + .build()) + .build()); + } + + /** + * Builds and returns the final structured results. + * + * @return Complete structured import results + */ + ImportResult getResults() { + return structuredResults.build(); + } + } + + /** + * Value object holding count information for import processing. Tracks valid/invalid records + * and created/updated content counts. + */ + private static class ImportCounts { + + private final int valid; + private final int invalid; + private final int created; + private final int updated; + + /** + * Creates a new ImportCounts instance. + * + * @param valid Count of valid records + * @param invalid Count of invalid records + * @param created Count of newly created content + * @param updated Count of updated content + */ + private ImportCounts(int valid, int invalid, int created, int updated) { + this.valid = valid; + this.invalid = invalid; + this.created = created; + this.updated = updated; + } + + /** + * Factory method to create an ImportCounts instance. + * + * @param valid Count of valid records + * @param invalid Count of invalid records + * @param created Count of newly created content + * @param updated Count of updated content + * @return New ImportCounts instance + */ + static ImportCounts of(int valid, int invalid, int created, int updated) { + return new ImportCounts(valid, invalid, created, updated); + } + + public int getValid() { + return valid; + } + + public int getInvalid() { + return invalid; + } + + public int getCreated() { + return created; + } + + public int getUpdated() { + return updated; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ImportCounts that = (ImportCounts) o; + return valid == that.valid && + invalid == that.invalid && + created == that.created && + updated == that.updated; + } + + @Override + public int hashCode() { + return Objects.hash(valid, invalid, created, updated); + } + } + } diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractContentSummary.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractContentSummary.java new file mode 100644 index 00000000000..afbf0f2839c --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractContentSummary.java @@ -0,0 +1,33 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * Immutable data structure that holds summary information about content processing results. + * This includes counts of created and updated content, as well as the content type identifier. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ContentSummary.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractContentSummary extends Serializable { + + /** + * @return The number of new content items created during the import process + */ + int created(); + + /** + * @return The number of existing content items updated during the import process + */ + int updated(); + + /** + * @return The identifier of the content type being processed + */ + String contentType(); + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportFileInfo.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportFileInfo.java new file mode 100644 index 00000000000..0e0e47dcf4b --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportFileInfo.java @@ -0,0 +1,33 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * Immutable data structure containing file processing information during import. + * Tracks the total number of rows in the file and how many were successfully parsed, + * along with header validation information. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportFileInfo.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportFileInfo extends Serializable { + + /** + * @return The total number of rows found in the import file + */ + int totalRows(); + + /** + * @return The number of rows successfully parsed from the import file + */ + int parsedRows(); + + /** + * @return Detailed information about the validated headers in the import file + */ + ImportHeaderInfo headerInfo(); +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderInfo.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderInfo.java new file mode 100644 index 00000000000..fa1772633d9 --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderInfo.java @@ -0,0 +1,43 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import java.util.Map; +import org.immutables.value.Value; + +/** + * Immutable data structure containing header validation information from the import file. + * Tracks valid, invalid, and missing headers, along with any additional validation details. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportHeaderInfo.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportHeaderInfo extends Serializable { + + /** + * @return The total number of headers found in the import file + */ + int totalHeaders(); + + /** + * @return Array of headers that were successfully validated against the content type + */ + String[] validHeaders(); + + /** + * @return Array of headers that did not match any content type fields + */ + String[] invalidHeaders(); + + /** + * @return Array of required content type fields not found in the headers + */ + String[] missingHeaders(); + + /** + * @return Additional validation details and metadata about the headers + */ + Map validationDetails(); +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderValidationResult.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderValidationResult.java new file mode 100644 index 00000000000..6bfe6eac4dd --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportHeaderValidationResult.java @@ -0,0 +1,36 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.List; +import java.util.Map; +import org.immutables.value.Value; + +/** + * Immutable data structure representing the complete results of header validation. + * Includes header information, validation messages, and contextual data needed for + * the import process. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportHeaderValidationResult.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportHeaderValidationResult { + + /** + * @return Detailed information about the validated headers + */ + ImportHeaderInfo headerInfo(); + + /** + * @return List of validation messages generated during header processing + */ + List messages(); + + /** + * @return Contextual information needed for the import process, such as processed headers, + * relationships, and field mappings + */ + Map context(); + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResult.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResult.java new file mode 100644 index 00000000000..22d6ab17822 --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResult.java @@ -0,0 +1,34 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import java.util.List; +import org.immutables.value.Value; + +/** + * Immutable data structure representing the complete results of an import operation. + * Contains file information, data processing results, and validation messages. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportResult.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportResult extends Serializable { + + /** + * @return Information about the processed import file + */ + ImportFileInfo fileInfo(); + + /** + * @return Results of the data processing operation + */ + ImportResultData data(); + + /** + * @return List of validation and processing messages generated during import + */ + List messages(); + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResultData.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResultData.java new file mode 100644 index 00000000000..8ecc37f9bb0 --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportResultData.java @@ -0,0 +1,28 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * Immutable data structure that combines processing statistics and content summary information. + * This interface represents the complete data results of an import operation. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportResultData.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportResultData extends Serializable { + + /** + * @return Statistics about processed records including valid and invalid counts + */ + ProcessedData processed(); + + /** + * @return Summary information about created and updated content + */ + ContentSummary summary(); + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportValidationMessage.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportValidationMessage.java new file mode 100644 index 00000000000..663b7e49ddd --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractImportValidationMessage.java @@ -0,0 +1,73 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * Immutable data structure representing a validation message generated during the import process. + * Messages can be errors, warnings, or informational, and may include contextual information + * such as line numbers, field names, and invalid values. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ImportValidationMessage.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractImportValidationMessage extends Serializable { + + /** + * Enumeration of possible message types for validation results. + */ + enum ValidationMessageType { + /** + * Indicates a critical error that prevents successful processing + */ + ERROR, + /** + * Indicates a potential issue that doesn't prevent processing + */ + WARNING, + /** + * Provides additional information about the process + */ + INFO + } + + /** + * @return The type of validation message + */ + ValidationMessageType type(); + + /** + * @return Optional validation code identifying the specific type of message + */ + Optional code(); + + /** + * @return The human-readable validation message + */ + String message(); + + /** + * @return Optional line number in the import file where the issue was found + */ + Optional lineNumber(); + + /** + * @return Optional field name related to the validation message + */ + Optional field(); + + /** + * @return Additional contextual information about the validation + */ + Map context(); + + /** + * @return Optional value that failed validation + */ + Optional invalidValue(); +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractProcessedData.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractProcessedData.java new file mode 100644 index 00000000000..6d2a5854dad --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/AbstractProcessedData.java @@ -0,0 +1,28 @@ +package com.dotmarketing.util.importer; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * Immutable data structure containing counts of valid and invalid records processed + * during the import operation. + */ +@Value.Style(typeImmutable = "*", typeAbstract = "Abstract*") +@Value.Immutable +@JsonDeserialize(as = ProcessedData.class) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AbstractProcessedData extends Serializable { + + /** + * @return The number of records that passed validation and were processed successfully + */ + int valid(); + + /** + * @return The number of records that failed validation or could not be processed + */ + int invalid(); + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/HeaderValidationCodes.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/HeaderValidationCodes.java new file mode 100644 index 00000000000..1922feab08d --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/HeaderValidationCodes.java @@ -0,0 +1,37 @@ +package com.dotmarketing.util.importer; + +/** + * Enumeration of validation codes used to identify specific types of header validation issues + * during the import process. These codes provide a standardized way to categorize and handle + * different validation scenarios. + */ +public enum HeaderValidationCodes { + + /** Indicates a header that doesn't match any content type field */ + INVALID_HEADER, + /** Indicates a system-level header (e.g., Identifier, Workflow Action) */ + SYSTEM_HEADER, + /** Indicates malformed or unreadable header format */ + INVALID_HEADER_FORMAT, + /** Indicates a header name that appears more than once */ + DUPLICATE_HEADER, + /** Indicates not all required content type fields are present in headers */ + INCOMPLETE_HEADERS, + /** Indicates a required field is missing from the headers */ + REQUIRED_FIELD_MISSING, + /** Indicates no key fields were specified for content matching */ + NO_KEY_FIELDS, + /** Indicates a field marked as unique in the content type */ + UNIQUE_FIELD, + /** Indicates an invalid key field specification */ + INVALID_KEY_FIELD, + /** Indicates duplicate values found for unique fields */ + DUPLICATE_VALUES, + /** Indicates issues with language-specific headers */ + INVALID_LANGUAGE, + /** Indicates security-related validation failures */ + SECURITY_ERROR, + /** Indicates general processing errors during validation */ + PROCESSING_ERROR + +} diff --git a/dotCMS/src/main/java/com/dotmarketing/util/importer/ImportResultConverter.java b/dotCMS/src/main/java/com/dotmarketing/util/importer/ImportResultConverter.java new file mode 100644 index 00000000000..e06f4838dc6 --- /dev/null +++ b/dotCMS/src/main/java/com/dotmarketing/util/importer/ImportResultConverter.java @@ -0,0 +1,175 @@ +package com.dotmarketing.util.importer; + +import com.dotmarketing.util.importer.AbstractImportValidationMessage.ValidationMessageType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Converter utility to maintain backward compatibility during the migration from legacy + * HashMap-based import results to the new structured ImportResult format. This class handles + * converting the new structured format back to the legacy format that existing code expects. + *

+ * The legacy format uses a HashMap with standard keys: + *

    + *
  • "warnings" - List of warning messages
    • + *
  • "errors" - List of error messages
    • + *
  • "messages" - List of informational messages
    • + *
  • "counters" - List of counter values in format "key=value"
    • + *
  • "results" - List of summary messages about the operation
    • + *
  • "validHeaders" - List of valid header names
    • + *
  • "invalidHeaders" - List of invalid header names
    • + *
  • "missingHeaders" - List of required headers that were not found
    • + *
  • "headerValidation" - List of header validation details
    • + *
+ */ +public class ImportResultConverter { + + /** + * Converts from the new structured ImportResult format to the legacy HashMap format. This + * method ensures backward compatibility by formatting the structured data into the string-based + * format expected by legacy code. + * + * @param result The structured ImportResult to convert + * @return A HashMap containing the legacy format results with standard keys for warnings, + * errors, messages, counters, and other import-related data + * @throws IllegalArgumentException if the result parameter is null + */ + public static Map> toLegacyFormat(ImportResult result) { + Map> legacyResults = new HashMap<>(); + + // Convert messages to legacy format by type + Map> messagesByType = + convertMessagesToLegacyFormat(result.messages()); + + legacyResults.put("warnings", messagesByType.get(ValidationMessageType.WARNING)); + legacyResults.put("errors", messagesByType.get(ValidationMessageType.ERROR)); + legacyResults.put("messages", messagesByType.get(ValidationMessageType.INFO)); + + // Add counters + List counters = new ArrayList<>(); + ImportResultData data = result.data(); + counters.add("linesread=" + result.fileInfo().totalRows()); + counters.add("errors=" + messagesByType.get(ValidationMessageType.ERROR).size()); + counters.add("newContent=" + data.summary().created()); + counters.add("contentToUpdate=" + data.summary().updated()); + legacyResults.put("counters", counters); + + // Add results summary + List results = new ArrayList<>(); + results.add(data.summary().created() + " new \"" + data.summary().contentType() + + "\" were created"); + if (data.summary().updated() > 0) { + results.add(data.summary().updated() + " \"" + data.summary().contentType() + + "\" contentlets updated"); + } + legacyResults.put("results", results); + + // Add header validation info if present + addHeaderValidationToLegacy(result.fileInfo().headerInfo(), legacyResults); + + return legacyResults; + } + + /** + * Converts validation messages from the structured format to legacy format, organizing them by + * message type. Each message is formatted to include line numbers, field names, and invalid + * values where applicable. + * + * @param messages The list of structured validation messages to convert + * @return A map where keys are validation message types and values are lists of formatted + * message strings + */ + private static Map> convertMessagesToLegacyFormat( + List messages) { + + Map> result = Arrays.stream( + ValidationMessageType.values()) + .collect(Collectors.toMap( + type -> type, + type -> new ArrayList<>() + )); + + if (messages != null) { + messages.forEach(message -> + result.computeIfAbsent(message.type(), k -> new ArrayList<>()) + .add(formatMessage(message))); + } + + return result; + } + + /** + * Formats a single validation message into a human-readable string format. The resulting string + * includes: + *
    + *
  • Line number (if present): "Line X: "
    • + *
  • Field name (if present): "Field 'X': "
    • + *
  • The main message
    • + *
  • Invalid value (if present): " (value: 'X')"
    • + *
+ * + * @param message The validation message to format + * @return A formatted string representation of the message + */ + private static String formatMessage(ImportValidationMessage message) { + StringBuilder sb = new StringBuilder(); + + // Add line number if present + message.lineNumber().ifPresent(line -> + sb.append("Line ").append(line).append(": ")); + + // Add field if present + message.field().ifPresent(field -> + sb.append("Field '").append(field).append("': ")); + + // Add main message + sb.append(message.message()); + + // Add any invalid value + message.invalidValue().ifPresent(value -> + sb.append(" (value: '").append(value).append("')")); + + return sb.toString(); + } + + /** + * Adds header validation information to the legacy results map. This includes arrays of valid, + * invalid, and missing headers, as well as any additional validation details stored in the + * headerInfo. + * + * @param headerInfo The structured header validation information + * @param legacyResults The legacy format results map to update + */ + private static void addHeaderValidationToLegacy(ImportHeaderInfo headerInfo, + Map> legacyResults) { + + if (headerInfo == null || legacyResults == null) { + return; + } + + // Add header arrays + legacyResults.put("validHeaders", + Arrays.asList(headerInfo.validHeaders() != null ? headerInfo.validHeaders() + : new String[0])); + legacyResults.put("invalidHeaders", + Arrays.asList(headerInfo.invalidHeaders() != null ? headerInfo.invalidHeaders() + : new String[0])); + legacyResults.put("missingHeaders", + Arrays.asList(headerInfo.missingHeaders() != null ? headerInfo.missingHeaders() + : new String[0])); + + // Add validation details + List headerValidation = new ArrayList<>(); + Map details = headerInfo.validationDetails(); + if (details != null) { + details.forEach((key, value) -> + headerValidation.add(key + "=" + value)); + } + legacyResults.put("headerValidation", headerValidation); + } + +} \ No newline at end of file