Introduce evaluated & rejected events metric

NationalSecurityAgency · May 13, 2024 · 798539b · 798539b
1 parent b22364e
commit 798539b
Show file tree

Hide file tree

Showing 15 changed files with 108 additions and 5 deletions.
diff --git a/api/pom.xml b/api/pom.xml
@@ -8,7 +8,7 @@
         <relativePath>../../../microservice-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-api</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>3.0.11-SNAPSHOT</version>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <scm>
         <connection>scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git</connection>

diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java
@@ -668,6 +668,10 @@ public int getFieldNumber(String name) {
     @XmlElement
     protected long seekCount = 0;
     @XmlElement
+    protected long evaluatedCount = 0;
+    @XmlElement
+    protected long rejectedCount = 0;
+    @XmlElement
     protected long yieldCount = 0L;
     @XmlElement
     @XmlJavaTypeAdapter(StringMapAdapter.class)
@@ -891,6 +895,22 @@ public void setSeekCount(long seekCount) {
         this.seekCount = seekCount;
     }
 
+    public long getEvaluatedCount() {
+        return evaluatedCount;
+    }
+
+    public void setEvaluatedCount(long evaluatedCount) {
+        this.evaluatedCount = evaluatedCount;
+    }
+
+    public long getRejectedCount() {
+        return rejectedCount;
+    }
+
+    public void setRejectedCount(long rejectedCount) {
+        this.rejectedCount = rejectedCount;
+    }
+
     public long getYieldCount() {
         return this.yieldCount;
     }

diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java
@@ -97,6 +97,8 @@ public QueryMetric(QueryMetric other) {
         this.sourceCount = other.sourceCount;
         this.nextCount = other.nextCount;
         this.seekCount = other.seekCount;
+        this.evaluatedCount = other.evaluatedCount;
+        this.rejectedCount = other.rejectedCount;
         this.yieldCount = other.yieldCount;
         this.versionMap = other.versionMap;
         this.docRanges = other.docRanges;
@@ -140,8 +142,9 @@ public int hashCode() {
                         .append(this.getHost()).append(this.getPageTimes()).append(this.getProxyServers()).append(this.getLifecycle())
                         .append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName())
                         .append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount())
-                        .append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime())
-                        .append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
+                        .append(this.getEvaluatedCount()).append(this.getRejectedCount()).append(this.getYieldCount()).append(this.getDocRanges())
+                        .append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime()).append(this.getPredictions()).append(this.getMarkings())
+                        .append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
     }
 
     @Override
@@ -166,6 +169,7 @@ public boolean equals(Object o) {
                             .append(this.getLifecycle(), other.getLifecycle()).append(this.getErrorMessage(), other.getErrorMessage())
                             .append(this.getErrorCode(), other.getErrorCode()).append(this.getSourceCount(), other.getSourceCount())
                             .append(this.getNextCount(), other.getNextCount()).append(this.getSeekCount(), other.getSeekCount())
+                            .append(this.getEvaluatedCount(), other.getEvaluatedCount()).append(this.getRejectedCount(), other.getRejectedCount())
                             .append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges())
                             .append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan())
                             .append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions())
@@ -207,6 +211,8 @@ public String toString() {
         buf.append(" Source Count: ").append(this.getSourceCount());
         buf.append(" NextCount: ").append(this.getNextCount());
         buf.append(" Seek Count: ").append(this.getSeekCount());
+        buf.append(" Evaluated Count: ").append(this.getEvaluatedCount());
+        buf.append(" Rejected Count: ").append(this.getRejectedCount());
         buf.append(" Yield Count: ").append(this.getYieldCount());
         buf.append(" Doc Ranges: ").append(this.getDocRanges());
         buf.append(" FI Ranges: ").append(this.getFiRanges());
@@ -403,6 +409,9 @@ public void writeTo(Output output, QueryMetric message) throws IOException {
                     output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true);
                 }
             }
+
+            output.writeInt64(39, message.evaluatedCount, false);
+            output.writeInt64(40, message.rejectedCount, false);
         }
 
         public void mergeFrom(Input input, QueryMetric message) throws IOException {
@@ -551,6 +560,12 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException {
                             message.versionMap.put(split[0], split[1]);
                         }
                         break;
+                    case 39:
+                        message.evaluatedCount = input.readInt64();
+                        break;
+                    case 40:
+                        message.rejectedCount = input.readInt64();
+                        break;
                     default:
                         input.handleUnknownField(number, this);
                         break;
@@ -637,6 +652,10 @@ public String getFieldName(int number) {
                     return "version";
                 case 38:
                     return "versionMap";
+                case 39:
+                    return "evaluatedCount";
+                case 40:
+                    return "rejectedCount";
                 default:
                     return null;
             }
@@ -688,6 +707,8 @@ public int getFieldNumber(String name) {
             fieldMap.put("predictions", 36);
             fieldMap.put("version", 37);
             fieldMap.put("versionMap", 38);
+            fieldMap.put("evaluatedCount", 39);
+            fieldMap.put("rejectedCount", 40);
         }
     };
 

diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java
@@ -109,6 +109,14 @@ public String getSeekCountStr() {
         return nf.format(seekCount);
     }
 
+    public String getEvaluatedCountStr() {
+        return nf.format(evaluatedCount);
+    }
+
+    public String getRejectedCountStr() {
+        return nf.format(rejectedCount);
+    }
+
     public String getYieldCountStr() {
         return nf.format(yieldCount);
     }

diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java
@@ -52,6 +52,10 @@ public interface QueryMetricModelFormat {
 
     String getSeekCountStr();
 
+    String getEvaluatedCountStr();
+
+    String getRejectedCountStr();
+
     String getYieldCountStr();
 
     String getVersionStr();

diff --git a/api/src/main/resources/templates/querymetric-horizontal.html b/api/src/main/resources/templates/querymetric-horizontal.html
@@ -57,6 +57,8 @@ <h1>Query Metrics</h1>
                 <th>Sources</th>
                 <th>Next Calls</th>
                 <th>Seek Calls</th>
+                <th>Evaluated Events</th>
+                <th>Rejected Events</th>
                 <th>Yield Count</th>
                 <th>Version</th>
                 <th>Total Page Time (ms)</th>
@@ -97,6 +99,8 @@ <h1>Query Metrics</h1>
                 <td th:text="${metric.sourceCountStr}"></td>
                 <td th:text="${metric.nextCountStr}"></td>
                 <td th:text="${metric.seekCountStr}"></td>
+                <td th:text="${metric.evaluatedCountStr}"></td>
+                <td th:text="${metric.rejectedCountStr}"></td>
                 <td th:text="${metric.yieldCountStr}"></td>
                 <td th:text="${metric.versionStr}"></td>
                 <td th:text="${metric.totalPageTimeStr}"></td>

diff --git a/api/src/main/resources/templates/querymetric.html b/api/src/main/resources/templates/querymetric.html
@@ -77,6 +77,8 @@ <h1>Query Metrics</h1>
             <tr><td>Sources</td><td th:text="${metric.sourceCountStr}"></td></tr>
             <tr class="highlight"><td>Next Calls</td><td th:text="${metric.nextCountStr}"></td>
             <tr><td>Seek Calls</td><td th:text="${metric.seekCountStr}"></td></tr>
+            <tr class="highlight"><td>Evaluated Events</td><td th:text="${metric.evaluatedCountStr}"></td>
+            <tr><td>Rejected Events</td><td th:text="${metric.rejectedCountStr}"></td></tr>
             <tr class="highlight"><td>Yield Count</td><td th:text="${metric.yieldCountStr}"></td></tr>
             <tr><td>Total Page Time (ms)</td><td th:text="${metric.totalPageTimeStr}"></td></tr>
             <tr class="highlight"><td>Total Page Call Time (ms)</td><td th:text="${metric.totalPageCallTimeStr}"></td></tr>

diff --git a/pom.xml b/pom.xml
@@ -8,7 +8,7 @@
         <relativePath>../../microservice-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-parent</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>3.0.11-SNAPSHOT</version>
     <packaging>pom</packaging>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <modules>

diff --git a/service/pom.xml b/service/pom.xml
@@ -8,7 +8,7 @@
         <relativePath>../../../microservice-service-parent/pom.xml</relativePath>
     </parent>
     <artifactId>query-metric-service</artifactId>
-    <version>3.0.10-SNAPSHOT</version>
+    <version>3.0.11-SNAPSHOT</version>
     <description>DATAWAVE Query Metric Microservice</description>
     <url>https://code.nsa.gov/datawave-query-metric-service</url>
     <scm>

diff --git a/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java b/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java
@@ -41,6 +41,8 @@ public Long process(Map.Entry<String,QueryMetricUpdateHolder> entry) {
             storedHolder.addValue("sourceCount", updatedMetric.getSourceCount());
             storedHolder.addValue("nextCount", updatedMetric.getNextCount());
             storedHolder.addValue("seekCount", updatedMetric.getSeekCount());
+            storedHolder.addValue("evaluatedCount", updatedMetric.getEvaluatedCount());
+            storedHolder.addValue("rejectedCount", updatedMetric.getRejectedCount());
             storedHolder.addValue("yieldCount", updatedMetric.getYieldCount());
             storedHolder.addValue("docRanges", updatedMetric.getDocRanges());
             storedHolder.addValue("fiRanges", updatedMetric.getFiRanges());

diff --git a/.../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java b/.../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java
@@ -94,6 +94,7 @@ public class QueryMetricHandlerProperties {
             "END_DATE",
             "ERROR_CODE",
             "ERROR_MESSAGE",
+            "EVALUATED_COUNT",
             "FI_RANGES",
             "HOST",
             "LIFECYCLE",
@@ -112,6 +113,7 @@ public class QueryMetricHandlerProperties {
             "QUERY_LOGIC",
             "QUERY_NAME",
             "QUERY_TYPE",
+            "REJECTED_COUNT",
             "SEEK_COUNT",
             "SETUP_TIME",
             "SOURCE_COUNT",
@@ -142,9 +144,11 @@ public class QueryMetricHandlerProperties {
             "CREATE_CALL_TIME",
             "DOC_RANGES",
             "ELAPSED_TIME",
+            "EVALUATED_COUNT",
             "FI_RANGES",
             "LOGIN_TIME",
             "NEXT_COUNT",
+            "REJECTED_COUNT",
             "SEEK_COUNT",
             "SETUP_TIME",
             "SOURCE_COUNT",

diff --git a/.../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/.../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java
@@ -215,6 +215,9 @@ public Multimap<String,String> getEventFieldsToWrite(T updated, T stored) {
             if (isChanged(updated.getErrorMessage(), stored == null ? null : stored.getErrorMessage())) {
                 fields.put("ERROR_MESSAGE", updated.getErrorMessage());
             }
+            if (isChanged(updated.getEvaluatedCount(), stored == null ? -1 : stored.getEvaluatedCount())) {
+                fields.put("EVALUATED_COUNT", Long.toString(updated.getEvaluatedCount()));
+            }
             if (isChanged(updated.getFiRanges(), stored == null ? -1 : stored.getFiRanges())) {
                 fields.put("FI_RANGES", Long.toString(updated.getFiRanges()));
             }
@@ -319,6 +322,9 @@ public Multimap<String,String> getEventFieldsToWrite(T updated, T stored) {
                     fields.put("QUERY_TYPE", updated.getQueryType());
                 }
             }
+            if (isChanged(updated.getRejectedCount(), stored == null ? -1 : stored.getRejectedCount())) {
+                fields.put("REJECTED_COUNT", Long.toString(updated.getRejectedCount()));
+            }
             if (isFirstWrite(updated.getSetupTime(), stored == null ? 0 : stored.getSetupTime(), 0)) {
                 fields.put("SETUP_TIME", Long.toString(updated.getSetupTime()));
             }
@@ -385,6 +391,9 @@ public Multimap<String,String> getEventFieldsToDelete(T updated, T stored) {
                 if (isChanged(updated.getElapsedTime(), stored.getElapsedTime())) {
                     fields.put("ELAPSED_TIME", Long.toString(stored.getElapsedTime()));
                 }
+                if (isChanged(updated.getEvaluatedCount(), stored.getEvaluatedCount())) {
+                    fields.put("EVALUATED_COUNT", Long.toString(stored.getEvaluatedCount()));
+                }
                 if (isChanged(updated.getFiRanges(), stored.getFiRanges())) {
                     fields.put("FI_RANGES", Long.toString(stored.getFiRanges()));
                 }
@@ -443,6 +452,9 @@ public Multimap<String,String> getEventFieldsToDelete(T updated, T stored) {
                         fields.put("PLAN", stored.getPlan());
                     }
                 }
+                if (isChanged(updated.getRejectedCount(), stored.getRejectedCount())) {
+                    fields.put("REJECTED_COUNT", Long.toString(stored.getRejectedCount()));
+                }
                 if (isChanged(updated.getSeekCount(), stored.getSeekCount())) {
                     fields.put("SEEK_COUNT", Long.toString(stored.getSeekCount()));
                 }

diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java
@@ -182,13 +182,17 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy
                 combinedMetric.setSourceCount(combinedMetric.getSourceCount() + updatedQueryMetric.getSourceCount());
                 combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount());
                 combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount());
+                combinedMetric.setEvaluatedCount(combinedMetric.getEvaluatedCount() + updatedQueryMetric.getEvaluatedCount());
+                combinedMetric.setRejectedCount(combinedMetric.getRejectedCount() + updatedQueryMetric.getRejectedCount());
                 combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount());
                 combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges());
                 combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges());
             } else {
                 combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount());
                 combinedMetric.setNextCount(updatedQueryMetric.getNextCount());
                 combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount());
+                combinedMetric.setEvaluatedCount(updatedQueryMetric.getEvaluatedCount());
+                combinedMetric.setRejectedCount(updatedQueryMetric.getRejectedCount());
                 combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount());
                 combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges());
                 combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges());

diff --git a/...src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/...src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java
@@ -506,6 +506,15 @@ public T toMetric(EventBase event) {
                         m.setErrorCode(fieldValue);
                     } else if (fieldName.equals("ERROR_MESSAGE")) {
                         m.setErrorMessage(fieldValue);
+                    } else if (fieldName.equals("EVALUATED_COUNT")) {
+                        try {
+                            long l = Long.parseLong(fieldValue);
+                            if (l > m.getEvaluatedCount()) {
+                                m.setEvaluatedCount(l);
+                            }
+                        } catch (Exception e) {
+                            log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
+                        }
                     } else if (fieldName.equals("FI_RANGES")) {
                         try {
                             long l = Long.parseLong(fieldValue);
@@ -614,6 +623,15 @@ public T toMetric(EventBase event) {
                         m.setQueryName(fieldValue);
                     } else if (fieldName.equals("QUERY_TYPE")) {
                         m.setQueryType(fieldValue);
+                    } else if (fieldName.equals("REJECTED_COUNT")) {
+                        try {
+                            long l = Long.parseLong(fieldValue);
+                            if (l > m.getRejectedCount()) {
+                                m.setRejectedCount(l);
+                            }
+                        } catch (Exception e) {
+                            log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
+                        }
                     } else if (fieldName.equals("SEEK_COUNT")) {
                         try {
                             long l = Long.parseLong(fieldValue);

diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java
@@ -161,6 +161,8 @@ public void store(QueryMetricUpdateHolder<T> queryMetricUpdate) throws Exception
                     updatedMetric.setSourceCount(queryMetricUpdate.getValue("sourceCount"));
                     updatedMetric.setNextCount(queryMetricUpdate.getValue("nextCount"));
                     updatedMetric.setSeekCount(queryMetricUpdate.getValue("seekCount"));
+                    updatedMetric.setEvaluatedCount(queryMetricUpdate.getValue("evaluatedCount"));
+                    updatedMetric.setRejectedCount(queryMetricUpdate.getValue("rejectedCount"));
                     updatedMetric.setYieldCount(queryMetricUpdate.getValue("yieldCount"));
                     updatedMetric.setDocRanges(queryMetricUpdate.getValue("docRanges"));
                     updatedMetric.setFiRanges(queryMetricUpdate.getValue("fiRanges"));
@@ -204,6 +206,8 @@ public void store(QueryMetricUpdateHolder<T> queryMetricUpdate) throws Exception
                     queryMetricUpdate.getMetric().setSourceCount(updatedMetric.getSourceCount());
                     queryMetricUpdate.getMetric().setNextCount(updatedMetric.getNextCount());
                     queryMetricUpdate.getMetric().setSeekCount(updatedMetric.getSeekCount());
+                    queryMetricUpdate.getMetric().setEvaluatedCount(updatedMetric.getEvaluatedCount());
+                    queryMetricUpdate.getMetric().setRejectedCount(updatedMetric.getRejectedCount());
                     queryMetricUpdate.getMetric().setYieldCount(updatedMetric.getYieldCount());
                     queryMetricUpdate.getMetric().setDocRanges(updatedMetric.getDocRanges());
                     queryMetricUpdate.getMetric().setFiRanges(updatedMetric.getFiRanges());