Feature/query microservices (#18)

* updated package locations of various classes * Updated datawave to 5.4.0-SNAPSHOT * bumped datawave version. * bumped datawave version * Updated to use jexl3 * Moved Query, QueryImpl to query-api * Updated with latest changes from integration * removed banned dependency * bumped release version * formatting * bumped versions for some modules * Updated with latest changes from main/integration * fixed confirm ack configuration, and updated query client response type. * updated with latest from integration * PR feedback * PR feedback * Update query-metric service to generate HTML response via Spring MVC (#6) * replaced apache commons base64 encoder/decoder with native java (#17) * datawave version to 6.9.3 * Release datawave-query-metric api/service 3.0.9 * Set version to 3.0.10-SNAPSHOT * Fixed setter typo * Update query-metric service to generate HTML response via Spring MVC (#6) * replaced apache commons base64 encoder/decoder with native java (#17) * datawave version to 6.9.3 * Release datawave-query-metric api/service 3.0.9 * Set version to 3.0.10-SNAPSHOT * fix typo in QueryGeometryHandler --------- Co-authored-by: Kevin Rathbun <[email protected]> Co-authored-by: Chris <[email protected]> Co-authored-by: Bill Oley <[email protected]>
NationalSecurityAgency · May 20, 2024 · 4049b64 · 4049b64
1 parent 6c5241b
commit 4049b64
Show file tree

Hide file tree

Showing 22 changed files with 103 additions and 69 deletions.
diff --git a/api/pom.xml b/api/pom.xml
@@ -4,11 +4,11 @@
     <parent>
         <groupId>gov.nsa.datawave.microservice</groupId>
         <artifactId>datawave-microservice-parent</artifactId>
-        <version>3.0.4</version>
+        <version>4.0.0-SNAPSHOT</version>
         <relativePath>../../../microservice-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-api</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>4.0.0-SNAPSHOT</version>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <scm>
         <connection>scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git</connection>
@@ -19,15 +19,16 @@
     <properties>
         <datawave.webservice.namespace>http://webservice.datawave.nsa/v1</datawave.webservice.namespace>
         <spotbugs.excludes.file>${basedir}/spotbugs-exclude.xml</spotbugs.excludes.file>
-        <version.accumulo-utils>3.0.1</version.accumulo-utils>
-        <version.base-rest-responses>3.0.0</version.base-rest-responses>
+        <version.accumulo-utils>4.0.0-SNAPSHOT</version.accumulo-utils>
+        <version.base-rest-responses>4.0.0-SNAPSHOT</version.base-rest-responses>
         <version.commons-lang3>3.12.0</version.commons-lang3>
         <version.commons-text>1.9</version.commons-text>
-        <version.datawave>6.9.3</version.datawave>
+        <version.datawave>7.0.0-SNAPSHOT</version.datawave>
         <version.glassfish>2.3.6</version.glassfish>
         <version.guava>31.1-jre</version.guava>
         <version.jackson>2.12.5</version.jackson>
         <version.jaxb-api>2.3.1</version.jaxb-api>
+        <version.microservice.query-api>1.0.0-SNAPSHOT</version.microservice.query-api>
         <version.protobuf-java>3.7.1</version.protobuf-java>
         <version.protostuff>1.6.2</version.protostuff>
         <version.slf4j>1.7.36</version.slf4j>
@@ -66,6 +67,17 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>gov.nsa.datawave.microservice</groupId>
+                <artifactId>query-api</artifactId>
+                <version>${version.microservice.query-api}</version>
+                <exclusions>
+                    <exclusion>
+                        <artifactId>*</artifactId>
+                        <groupId>*</groupId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
             <dependency>
                 <groupId>gov.nsa.datawave.webservices</groupId>
                 <artifactId>datawave-ws-client</artifactId>
@@ -157,6 +169,10 @@
             <groupId>gov.nsa.datawave.microservice</groupId>
             <artifactId>base-rest-responses</artifactId>
         </dependency>
+        <dependency>
+            <groupId>gov.nsa.datawave.microservice</groupId>
+            <artifactId>query-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>gov.nsa.datawave.webservices</groupId>
             <artifactId>datawave-ws-client</artifactId>

diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java
@@ -34,8 +34,8 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import datawave.marking.MarkingFunctions;
-import datawave.webservice.query.Query;
-import datawave.webservice.query.QueryImpl.Parameter;
+import datawave.microservice.query.Query;
+import datawave.microservice.query.QueryImpl.Parameter;
 import datawave.webservice.query.exception.QueryException;
 import datawave.webservice.query.result.event.HasMarkings;
 import io.protostuff.Input;

diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java
@@ -24,8 +24,8 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.Lists;
 
-import datawave.webservice.query.Query;
-import datawave.webservice.query.QueryImpl.Parameter;
+import datawave.microservice.query.Query;
+import datawave.microservice.query.QueryImpl.Parameter;
 import datawave.webservice.query.result.event.HasMarkings;
 import io.protostuff.Input;
 import io.protostuff.Message;

diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java
@@ -6,7 +6,7 @@
 import java.util.List;
 import java.util.Set;
 
-import datawave.webservice.query.QueryImpl;
+import datawave.microservice.query.QueryImpl;
 
 public interface QueryMetricModelFormat {
 

diff --git a/pom.xml b/pom.xml
@@ -4,11 +4,11 @@
     <parent>
         <groupId>gov.nsa.datawave.microservice</groupId>
         <artifactId>datawave-microservice-parent</artifactId>
-        <version>3.0.4</version>
+        <version>4.0.0-SNAPSHOT</version>
         <relativePath>../../microservice-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-parent</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>4.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <modules>

diff --git a/service/pom.xml b/service/pom.xml
@@ -4,11 +4,11 @@
     <parent>
         <groupId>gov.nsa.datawave.microservice</groupId>
         <artifactId>datawave-microservice-service-parent</artifactId>
-        <version>4.0.4</version>
+        <version>5.0.0-SNAPSHOT</version>
         <relativePath>../../../microservice-service-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-service</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>4.0.0-SNAPSHOT</version>
     <description>DATAWAVE Query Metric Microservice</description>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <scm>
@@ -20,19 +20,19 @@
     <properties>
         <spotbugs.excludes.file>${basedir}/spotbugs-exclude.xml</spotbugs.excludes.file>
         <start-class>datawave.microservice.querymetric.QueryMetricService</start-class>
-        <version.accumulo-utils>3.0.1</version.accumulo-utils>
-        <version.base-rest-responses>3.0.0</version.base-rest-responses>
+        <version.accumulo-utils>4.0.0-SNAPSHOT</version.accumulo-utils>
+        <version.base-rest-responses>4.0.0-SNAPSHOT</version.base-rest-responses>
         <version.cache-api>1.1.1</version.cache-api>
-        <version.datawave>6.9.3</version.datawave>
+        <version.datawave>7.0.0-SNAPSHOT</version.datawave>
         <version.hazelcast-kubernetes>2.2.3</version.hazelcast-kubernetes>
-        <version.in-memory-accumulo>3.0.1</version.in-memory-accumulo>
-        <version.microservice.accumulo-api>3.0.0</version.microservice.accumulo-api>
-        <version.microservice.hazelcast>3.0.0</version.microservice.hazelcast>
-        <version.microservice.query-metric-api>${project.version}</version.microservice.query-metric-api>
-        <version.microservice.starter>3.0.0</version.microservice.starter>
-        <version.microservice.starter-datawave-query-metric>2.0.0</version.microservice.starter-datawave-query-metric>
-        <version.microservice.starter-metadata>2.0.0</version.microservice.starter-metadata>
-        <version.microservice.type-utils>2.0.2</version.microservice.type-utils>
+        <version.in-memory-accumulo>4.0.0-SNAPSHOT</version.in-memory-accumulo>
+        <version.microservice.accumulo-api>4.0.0-SNAPSHOT</version.microservice.accumulo-api>
+        <version.microservice.hazelcast>4.0.0-SNAPSHOT</version.microservice.hazelcast>
+        <version.microservice.query-metric-api>4.0.0-SNAPSHOT</version.microservice.query-metric-api>
+        <version.microservice.starter>4.0.0-SNAPSHOT</version.microservice.starter>
+        <version.microservice.starter-datawave-query-metric>3.0.0-SNAPSHOT</version.microservice.starter-datawave-query-metric>
+        <version.microservice.starter-metadata>3.0.0-SNAPSHOT</version.microservice.starter-metadata>
+        <version.microservice.type-utils>3.0.0-SNAPSHOT</version.microservice.type-utils>
         <version.protobuf-java>3.7.1</version.protobuf-java>
         <version.validation-api>2.0.2</version.validation-api>
         <version.webjars.bootstrap>4.3.1</version.webjars.bootstrap>
@@ -89,6 +89,10 @@
                         <artifactId>slf4j-reload4j</artifactId>
                         <groupId>org.slf4j</groupId>
                     </exclusion>
+                    <exclusion>
+                        <artifactId>reload4j</artifactId>
+                        <groupId>ch.qos.reload4j</groupId>
+                    </exclusion>
                 </exclusions>
             </dependency>
             <dependency>
@@ -237,6 +241,14 @@
                         <artifactId>avro</artifactId>
                         <groupId>org.apache.avro</groupId>
                     </exclusion>
+                    <exclusion>
+                        <artifactId>reload4j</artifactId>
+                        <groupId>ch.qos.reload4j</groupId>
+                    </exclusion>
+                    <exclusion>
+                        <artifactId>avro</artifactId>
+                        <groupId>org.apache.avro</groupId>
+                    </exclusion>
                 </exclusions>
             </dependency>
             <dependency>
@@ -280,6 +292,10 @@
                 <artifactId>zookeeper</artifactId>
                 <version>${version.zookeeper}</version>
                 <exclusions>
+                    <exclusion>
+                        <artifactId>*</artifactId>
+                        <groupId>ch.qos.logback</groupId>
+                    </exclusion>
                     <exclusion>
                         <artifactId>slf4j-log4j12</artifactId>
                         <groupId>org.slf4j</groupId>

diff --git a/service/src/main/java/datawave/microservice/querymetric/config/AccumuloConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/AccumuloConfiguration.java
@@ -8,11 +8,11 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Lazy;
 
+import datawave.core.common.connection.AccumuloClientPool;
+import datawave.core.common.connection.AccumuloClientPoolFactory;
 import datawave.microservice.config.accumulo.AccumuloProperties;
 import datawave.microservice.config.cluster.ClusterProperties;
 import datawave.microservice.querymetric.factory.WrappedAccumuloClientPoolFactory;
-import datawave.webservice.common.connection.AccumuloClientPool;
-import datawave.webservice.common.connection.AccumuloClientPoolFactory;
 
 @Configuration
 @EnableConfigurationProperties({AccumuloConfiguration.WarehouseClusterProperties.class})

diff --git a/...c/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/...c/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java
@@ -29,6 +29,8 @@
 import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
 import com.github.benmanes.caffeine.cache.CaffeineSpec;
 
+import datawave.core.common.connection.AccumuloClientPool;
+import datawave.core.query.result.event.DefaultResponseObjectFactory;
 import datawave.marking.MarkingFunctions;
 import datawave.microservice.http.converter.html.BannerProvider;
 import datawave.microservice.querymetric.BaseQueryMetric;
@@ -55,8 +57,6 @@
 import datawave.query.util.DateIndexHelperFactory;
 import datawave.query.util.TypeMetadataHelper;
 import datawave.security.authorization.JWTTokenHandler;
-import datawave.webservice.common.connection.AccumuloClientPool;
-import datawave.webservice.query.result.event.DefaultResponseObjectFactory;
 import datawave.webservice.query.result.event.ResponseObjectFactory;
 
 @Configuration

diff --git a/.../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java b/.../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java
@@ -76,9 +76,9 @@ public class QueryMetricHandlerProperties {
 
     protected boolean useRemoteQuery = true;
     protected long remoteAuthTimeout = 1L;
-    protected TimeUnit remoteAuthTimeUnit = TimeUnit.MINUTES;
+    protected TimeUnit remoteAuthTimeoutUnit = TimeUnit.MINUTES;
     protected long remoteQueryTimeout = 1L;
-    protected TimeUnit remoteQueryTimeUnit = TimeUnit.MINUTES;
+    protected TimeUnit remoteQueryTimeoutUnit = TimeUnit.MINUTES;
 
     protected String npeOuEntries;
     protected String subjectDnPattern;
@@ -519,39 +519,39 @@ public long getRemoteAuthTimeout() {
     }
 
     public long getRemoteAuthTimeoutMillis() {
-        return remoteAuthTimeUnit.toMillis(remoteAuthTimeout);
+        return remoteAuthTimeoutUnit.toMillis(remoteAuthTimeout);
     }
 
     public void setRemoteAuthTimeout(long remoteAuthTimeout) {
         this.remoteAuthTimeout = remoteAuthTimeout;
     }
 
-    public TimeUnit getRemoteAuthTimeUnit() {
-        return remoteAuthTimeUnit;
+    public TimeUnit getRemoteAuthTimeoutUnit() {
+        return remoteAuthTimeoutUnit;
     }
 
-    public void setRemoteAuthTimeUnit(TimeUnit remoteAuthTimeUnit) {
-        this.remoteAuthTimeUnit = remoteAuthTimeUnit;
+    public void setRemoteAuthTimeoutUnit(TimeUnit remoteAuthTimeoutUnit) {
+        this.remoteAuthTimeoutUnit = remoteAuthTimeoutUnit;
     }
 
     public long getRemoteQueryTimeout() {
         return remoteQueryTimeout;
     }
 
     public long getRemoteQueryTimeoutMillis() {
-        return remoteQueryTimeUnit.toMillis(remoteQueryTimeout);
+        return remoteQueryTimeoutUnit.toMillis(remoteQueryTimeout);
     }
 
     public void setRemoteQueryTimeout(long remoteQueryTimeout) {
         this.remoteQueryTimeout = remoteQueryTimeout;
     }
 
-    public TimeUnit getRemoteQueryTimeUnit() {
-        return remoteQueryTimeUnit;
+    public TimeUnit getRemoteQueryTimeoutUnit() {
+        return remoteQueryTimeoutUnit;
     }
 
-    public void setRemoteQueryTimeUnit(TimeUnit remoteQueryTimeUnit) {
-        this.remoteQueryTimeUnit = remoteQueryTimeUnit;
+    public void setRemoteQueryTimeoutUnit(TimeUnit remoteQueryTimeoutUnit) {
+        this.remoteQueryTimeoutUnit = remoteQueryTimeoutUnit;
     }
 
     public String getNpeOuEntries() {

diff --git a/...main/java/datawave/microservice/querymetric/factory/WrappedAccumuloClientPoolFactory.java b/...main/java/datawave/microservice/querymetric/factory/WrappedAccumuloClientPoolFactory.java
@@ -8,7 +8,7 @@
 
 import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryInstance;
-import datawave.webservice.common.connection.AccumuloClientPoolFactory;
+import datawave.core.common.connection.AccumuloClientPoolFactory;
 import datawave.webservice.common.connection.WrappedAccumuloClient;
 
 /*

diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/AccumuloRecordWriter.java b/service/src/main/java/datawave/microservice/querymetric/handler/AccumuloRecordWriter.java
@@ -27,7 +27,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import datawave.webservice.common.connection.AccumuloClientPool;
+import datawave.core.common.connection.AccumuloClientPool;
 
 public class AccumuloRecordWriter extends RecordWriter<Text,Mutation> {
     private MultiTableBatchWriter mtbw = null;

diff --git a/.../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/.../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java
@@ -16,14 +16,15 @@
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 
+import datawave.core.query.util.QueryUtil;
 import datawave.ingest.data.config.NormalizedContentInterface;
 import datawave.ingest.data.config.NormalizedFieldAndValue;
 import datawave.ingest.data.config.ingest.CSVIngestHelper;
 import datawave.ingest.data.config.ingest.TermFrequencyIngestHelperInterface;
+import datawave.microservice.query.QueryImpl;
 import datawave.microservice.querymetric.BaseQueryMetric;
 import datawave.microservice.querymetric.BaseQueryMetric.PageMetric;
 import datawave.microservice.querymetric.BaseQueryMetric.Prediction;
-import datawave.webservice.query.util.QueryUtil;
 
 public class ContentQueryMetricsIngestHelper extends CSVIngestHelper implements TermFrequencyIngestHelperInterface {
 

diff --git a/...ain/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java b/...ain/java/datawave/microservice/querymetric/handler/LocalShardTableQueryMetricHandler.java
@@ -22,7 +22,12 @@
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
+import datawave.core.common.connection.AccumuloClientPool;
+import datawave.core.common.connection.AccumuloConnectionFactory;
+import datawave.core.query.logic.QueryLogic;
+import datawave.core.query.logic.QueryLogicTransformer;
 import datawave.marking.MarkingFunctions;
+import datawave.microservice.query.Query;
 import datawave.microservice.querymetric.BaseQueryMetric;
 import datawave.microservice.querymetric.QueryMetricFactory;
 import datawave.microservice.querymetric.config.QueryMetricHandlerProperties;
@@ -32,11 +37,6 @@
 import datawave.security.authorization.DatawavePrincipal;
 import datawave.security.authorization.DatawaveUser;
 import datawave.security.authorization.SubjectIssuerDNPair;
-import datawave.webservice.common.connection.AccumuloClientPool;
-import datawave.webservice.common.connection.AccumuloConnectionFactory;
-import datawave.webservice.query.Query;
-import datawave.webservice.query.logic.QueryLogic;
-import datawave.webservice.query.logic.QueryLogicTransformer;
 import datawave.webservice.query.runner.RunningQuery;
 import datawave.webservice.result.BaseQueryResponse;
 

diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java
@@ -5,10 +5,10 @@
 import java.util.Map;
 
 import datawave.microservice.authorization.user.DatawaveUserDetails;
+import datawave.microservice.query.Query;
 import datawave.microservice.querymetric.BaseQueryMetric;
 import datawave.microservice.querymetric.QueryMetricType;
 import datawave.microservice.querymetric.QueryMetricsSummaryResponse;
-import datawave.webservice.query.Query;
 
 public interface QueryMetricHandler<T extends BaseQueryMetric> {