Merge branch 'main' into task/collapseFrequencyValues

.github/workflows/tests.yaml

@@ -10,56 +10,8 @@ on:
     paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE']
 
 jobs:
-  # Runs the pom sorter and code formatter to ensure that the code
-  # is formatted and poms are sorted according to project rules. This
-  # will fail if the formatter makes any changes.
-  check-code-formatting:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v1
-    - name: Set up JDK 11
-      uses: actions/setup-java@v1
-      with:
-        java-version: 11
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-format-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-maven-format-
-          ${{ runner.os }}-maven-
-    - name: Format code
-      run: |
-        mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean formatter:format sortpom:sort -Pautoformat
-        git status
-        git diff-index --quiet HEAD || (echo "Error! There are modified files after formatting." && false)
-      env:
-        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
-        USER_NAME: ${{ secrets.USER_NAME }}
-        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
-
-  # Build the code and run the unit/integration tests.
-  build-and-test:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v1
-    - name: Set up JDK 11
-      uses: actions/setup-java@v1
-      with:
-        java-version: 11
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-maven-build-
-          ${{ runner.os }}-maven-format-
-          ${{ runner.os }}-maven-
-    - name: Build and Run Unit Tests
-      run: mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Ddist clean verify
-      env:
-        MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
-        USER_NAME: ${{ secrets.USER_NAME }}
-        ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+  call-reusable-workflow:
+    uses: nationalsecurityagency/datawave/.github/workflows/microservice-maven-tests.yaml@integration
+    secrets:
+      USER_NAME: ${{ secrets.USER_NAME }}
+      ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}

pom.xml

@@ -8,7 +8,7 @@
         <relativePath>../../../microservices/microservice-parent/pom.xml</relativePath>
     </parent>
     <artifactId>metadata-utils</artifactId>
-    <version>4.0.7-SNAPSHOT</version>
+    <version>4.0.9-SNAPSHOT</version>
     <url>https://code.nsa.gov/datawave-metadata-utils</url>
     <licenses>
         <license>

src/main/java/datawave/query/model/FieldMapping.java

@@ -5,6 +5,8 @@
 import java.util.Collections;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -71,6 +73,14 @@ public void validate() {
             if (fieldName == null || modelFieldName == null || columnVisibility == null) {
                 throw new IllegalArgumentException("Cannot have a model mapping with without all members: " + this);
             }
+            // If this is a forward mapping, it's possible that a regex pattern is supplied for the field name. Verify that the field name compiles.
+            if (direction == Direction.FORWARD) {
+                try {
+                    Pattern.compile(fieldName);
+                } catch (PatternSyntaxException e) {
+                    throw new IllegalArgumentException("Invalid regex pattern supplied for field name: " + fieldName, e);
+                }
+            }
         }
     }
 

src/main/java/datawave/query/util/AllFieldMetadataHelper.java

@@ -992,49 +992,49 @@ public Multimap<String,String> getIndexOnlyFields() throws TableNotFoundExceptio
 
         final Map<String,Multimap<Text,Text>> metadata = new HashMap<>();
 
-        Scanner bs = ScannerHelper.createScanner(accumuloClient, metadataTableName, auths);
-
-        // Fetch the 'e' and 'i' columns
-        bs.fetchColumnFamily(ColumnFamilyConstants.COLF_E);
-        bs.fetchColumnFamily(ColumnFamilyConstants.COLF_I);
-        bs.fetchColumnFamily(ColumnFamilyConstants.COLF_CI);
-
-        // For all keys in the DatawaveMetadata table
-        bs.setRange(new Range());
-
-        Iterator<Entry<Key,Value>> iterator = bs.iterator();
-        Set<String> compositeFields = Sets.newHashSet();
-        // Collect the results and put them into a Multimap
-        while (iterator.hasNext()) {
-            Entry<Key,Value> entry = iterator.next();
-            Key k = entry.getKey();
-            Text fieldName = k.getRow();
-            Text fieldType = k.getColumnFamily();
-            String dataType = getDatatype(k);
-            if (fieldType.equals(ColumnFamilyConstants.COLF_CI)) {
-                compositeFields.add(getCompositeFieldName(k));
-            }
+        try (Scanner bs = ScannerHelper.createScanner(accumuloClient, metadataTableName, auths)) {
+            // Fetch the 'e' and 'i' columns
+            bs.fetchColumnFamily(ColumnFamilyConstants.COLF_E);
+            bs.fetchColumnFamily(ColumnFamilyConstants.COLF_I);
+            bs.fetchColumnFamily(ColumnFamilyConstants.COLF_CI);
 
-            Multimap<Text,Text> md = metadata.get(dataType);
-            if (md == null) {
-                md = HashMultimap.create();
-                metadata.put(dataType, md);
+            // For all keys in the DatawaveMetadata table
+            bs.setRange(new Range());
+
+            Iterator<Entry<Key,Value>> iterator = bs.iterator();
+            Set<String> compositeFields = Sets.newHashSet();
+            // Collect the results and put them into a Multimap
+            while (iterator.hasNext()) {
+                Entry<Key,Value> entry = iterator.next();
+                Key k = entry.getKey();
+                Text fieldName = k.getRow();
+                Text fieldType = k.getColumnFamily();
+                String dataType = getDatatype(k);
+                if (fieldType.equals(ColumnFamilyConstants.COLF_CI)) {
+                    compositeFields.add(getCompositeFieldName(k));
+                }
 
+                Multimap<Text,Text> md = metadata.get(dataType);
+                if (md == null) {
+                    md = HashMultimap.create();
+                    metadata.put(dataType, md);
+
+                }
+                md.put(fieldName, fieldType);
             }
-            md.put(fieldName, fieldType);
-        }
-
-        // Find all of the fields which only have the 'i' column
-        for (String dataType : metadata.keySet()) {
-            for (Text fieldName : metadata.get(dataType).keySet()) {
-                Collection<Text> columns = metadata.get(dataType).get(fieldName);
-
-                if (1 == columns.size()) {
-                    Text c = columns.iterator().next();
+
+            // Find all of the fields which only have the 'i' column
+            for (String dataType : metadata.keySet()) {
+                for (Text fieldName : metadata.get(dataType).keySet()) {
+                    Collection<Text> columns = metadata.get(dataType).get(fieldName);
 
-                    if (c.equals(ColumnFamilyConstants.COLF_I)) {
-                        if (compositeFields.contains(fieldName.toString()) == false) {
-                            fields.put(dataType, fieldName.toString());
+                    if (1 == columns.size()) {
+                        Text c = columns.iterator().next();
+
+                        if (c.equals(ColumnFamilyConstants.COLF_I)) {
+                            if (compositeFields.contains(fieldName.toString()) == false) {
+                                fields.put(dataType, fieldName.toString());
+                            }
                         }
                     }
                 }

src/main/java/datawave/query/util/MetadataHelper.java

@@ -21,6 +21,7 @@
 import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -519,13 +520,22 @@ public QueryModel getQueryModel(String modelTableName, String modelName, Collect
                     if (!mapping.isFieldMapping()) {
                         queryModel.setModelFieldAttributes(mapping.getModelFieldName(), mapping.getAttributes());
                     } else if (mapping.getDirection() == Direction.FORWARD) {
-                        // Do not add a forward mapping entry
-                        // when the replacement does not exist in the database
+                        // If a direct match is found for the field in the database, add a forward mapping entry.
                         if (allFields.contains(mapping.getFieldName())) {
                             queryModel.addTermToModel(mapping.getModelFieldName(), mapping.getFieldName());
-                        } else if (log.isTraceEnabled()) {
-                            log.trace("Ignoring forward mapping of {} for {} because the metadata table has no reference to it", mapping.getFieldName(),
-                                            mapping.getModelFieldName());
+                        } else {
+                            // If a direct match was not found for the field name, it's possible that a regex pattern was supplied. Attempt to find matches
+                            // based off matching against the field name as a pattern.
+                            Pattern pattern = Pattern.compile(mapping.getFieldName());
+                            Set<String> matches = allFields.stream().filter(field -> pattern.matcher(field).matches()).collect(Collectors.toSet());
+                            if (!matches.isEmpty()) {
+                                matches.forEach(field -> queryModel.addTermToModel(mapping.getModelFieldName(), field));
+                            } else {
+                                if (log.isTraceEnabled()) {
+                                    log.trace("Ignoring forward mapping of {} for {} because the metadata table has no reference to it", mapping.getFieldName(),
+                                                    mapping.getModelFieldName());
+                                }
+                            }
                         }
                     } else {
                         queryModel.addTermToReverseModel(mapping.getFieldName(), mapping.getModelFieldName());
@@ -541,7 +551,7 @@ public QueryModel getQueryModel(String modelTableName, String modelName, Collect
                 log.trace("empty query model for {}", this);
             }
             if ("DatawaveMetadata".equals(modelTableName)) {
-                log.error("Query Model should not be empty...");
+                log.warn("Query Model {} has no reverse mappings", modelName);
             }
         }
 
@@ -555,7 +565,7 @@ public QueryModel getQueryModel(String modelTableName, String modelName, Collect
      * @return a list of query model names
      * @throws TableNotFoundException
      */
-    @Cacheable(value = "getQueryModelNames", key = "{#root.target.auths,#table}", cacheManager = "metadataHelperCacheManager")
+    @Cacheable(value = "getQueryModelNames", key = "{#root.target.auths,#modelTableName}", cacheManager = "metadataHelperCacheManager")
     public Set<String> getQueryModelNames(String modelTableName) throws TableNotFoundException {
         Preconditions.checkNotNull(modelTableName);
 

src/test/java/datawave/query/model/FieldMappingTest.java

@@ -0,0 +1,35 @@
+package datawave.query.model;
+
+import java.util.Collections;
+
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+public class FieldMappingTest {
+
+    /**
+     * Verify that creating a forward mapping with a regular field name does not result in any exceptions.
+     */
+    @Test
+    public void testForwardMappingWithPlainFieldName() {
+        Assertions.assertDoesNotThrow(() -> new FieldMapping("datatype", "DB_NAME", "FIELD_NAME", Direction.FORWARD, "ALL", Collections.emptySet()));
+    }
+
+    /**
+     * Verify that creating a forward mapping with an invalid pattern for the field name results in an exception.
+     */
+    @Test
+    public void testForwardMappingWithInvalidPatternFieldName() {
+        Assertions.assertThrows(IllegalArgumentException.class,
+                        () -> new FieldMapping("datatype", "[\\]", "FIELD_NAME", Direction.FORWARD, "ALL", Collections.emptySet()),
+                        "Invalid regex pattern supplied for field name: [\\]");
+    }
+
+    /**
+     * Verify that creating a forward mapping with a valid pattern for the field name does not result in an exception.
+     */
+    @Test
+    public void testForwardMappingWithValidPatternFieldName() {
+        Assertions.assertDoesNotThrow(() -> new FieldMapping("datatype", "DB_NAME.*", "FIELD_NAME", Direction.FORWARD, "ALL", Collections.emptySet()));
+    }
+}