From 5d7b1e84fe68d93c4d53f21405e3bc10be99a914 Mon Sep 17 00:00:00 2001 From: foster33 Date: Tue, 21 May 2024 12:51:41 +0000 Subject: [PATCH 1/2] update poms and rebase --- api/pom.xml | 4 +-- .../querymetric/BaseQueryMetric.java | 20 +++++++++++++++ .../microservice/querymetric/QueryMetric.java | 25 +++++++++++++++++-- pom.xml | 4 +-- service/pom.xml | 4 +-- .../MetricUpdateEntryProcessor.java | 2 ++ .../config/QueryMetricHandlerProperties.java | 4 +++ .../ContentQueryMetricsIngestHelper.java | 12 +++++++++ .../handler/QueryMetricCombiner.java | 4 +++ .../handler/ShardTableQueryMetricHandler.java | 18 +++++++++++++ .../persistence/AccumuloMapStore.java | 4 +++ 11 files changed, 93 insertions(+), 8 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 0a1d0036..47332ad2 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0.4 + 4.0.0 ../../../microservice-parent/pom.xml query-metric-api - 3.0.9-SNAPSHOT + 4.0.1-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index ad1dac9c..f8a03028 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -668,6 +668,10 @@ public int getFieldNumber(String name) { @XmlElement protected long seekCount = 0; @XmlElement + protected long evaluatedCount = 0; + @XmlElement + protected long rejectedCount = 0; + @XmlElement protected long yieldCount = 0L; @XmlElement @XmlJavaTypeAdapter(StringMapAdapter.class) @@ -891,6 +895,22 @@ public void setSeekCount(long seekCount) { this.seekCount = seekCount; } + public long getEvaluatedCount() { + return evaluatedCount; + } + + public void setEvaluatedCount(long evaluatedCount) { + this.evaluatedCount = evaluatedCount; + } + + public long getRejectedCount() { + return rejectedCount; + } + + public void setRejectedCount(long rejectedCount) { + this.rejectedCount = rejectedCount; + } + public long getYieldCount() { return this.yieldCount; } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index 59133d8b..f650a2c3 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -97,6 +97,8 @@ public QueryMetric(QueryMetric other) { this.sourceCount = other.sourceCount; this.nextCount = other.nextCount; this.seekCount = other.seekCount; + this.evaluatedCount = other.evaluatedCount; + this.rejectedCount = other.rejectedCount; this.yieldCount = other.yieldCount; this.versionMap = other.versionMap; this.docRanges = other.docRanges; @@ -140,8 +142,9 @@ public int hashCode() { .append(this.getHost()).append(this.getPageTimes()).append(this.getProxyServers()).append(this.getLifecycle()) .append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName()) .append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount()) - .append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime()) - .append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode(); + .append(this.getEvaluatedCount()).append(this.getRejectedCount()).append(this.getYieldCount()).append(this.getDocRanges()) + .append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime()).append(this.getPredictions()).append(this.getMarkings()) + .append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode(); } @Override @@ -166,6 +169,7 @@ public boolean equals(Object o) { .append(this.getLifecycle(), other.getLifecycle()).append(this.getErrorMessage(), other.getErrorMessage()) .append(this.getErrorCode(), other.getErrorCode()).append(this.getSourceCount(), other.getSourceCount()) .append(this.getNextCount(), other.getNextCount()).append(this.getSeekCount(), other.getSeekCount()) + .append(this.getEvaluatedCount(), other.getEvaluatedCount()).append(this.getRejectedCount(), other.getRejectedCount()) .append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges()) .append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan()) .append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions()) @@ -207,6 +211,8 @@ public String toString() { buf.append(" Source Count: ").append(this.getSourceCount()); buf.append(" NextCount: ").append(this.getNextCount()); buf.append(" Seek Count: ").append(this.getSeekCount()); + buf.append(" Evaluated Count: ").append(this.getEvaluatedCount()); + buf.append(" Rejected Count: ").append(this.getRejectedCount()); buf.append(" Yield Count: ").append(this.getYieldCount()); buf.append(" Doc Ranges: ").append(this.getDocRanges()); buf.append(" FI Ranges: ").append(this.getFiRanges()); @@ -403,6 +409,9 @@ public void writeTo(Output output, QueryMetric message) throws IOException { output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); } } + + output.writeInt64(39, message.evaluatedCount, false); + output.writeInt64(40, message.rejectedCount, false); } public void mergeFrom(Input input, QueryMetric message) throws IOException { @@ -551,6 +560,12 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { message.versionMap.put(split[0], split[1]); } break; + case 39: + message.evaluatedCount = input.readInt64(); + break; + case 40: + message.rejectedCount = input.readInt64(); + break; default: input.handleUnknownField(number, this); break; @@ -637,6 +652,10 @@ public String getFieldName(int number) { return "version"; case 38: return "versionMap"; + case 39: + return "evaluatedCount"; + case 40: + return "rejectedCount"; default: return null; } @@ -688,6 +707,8 @@ public int getFieldNumber(String name) { fieldMap.put("predictions", 36); fieldMap.put("version", 37); fieldMap.put("versionMap", 38); + fieldMap.put("evaluatedCount", 39); + fieldMap.put("rejectedCount", 40); } }; diff --git a/pom.xml b/pom.xml index 18ec61bb..9ebfe531 100644 --- a/pom.xml +++ b/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 3.0.4 + 4.0.0 ../../microservice-parent/pom.xml query-metric-parent - 3.0.9-SNAPSHOT + 4.0.1-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 86a082fb..f744e92f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -4,11 +4,11 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 4.0.4 + 5.0.0 ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.9-SNAPSHOT + 4.0.1-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service diff --git a/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java b/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java index 08c4cc7b..987adbe1 100644 --- a/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java +++ b/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java @@ -41,6 +41,8 @@ public Long process(Map.Entry entry) { storedHolder.addValue("sourceCount", updatedMetric.getSourceCount()); storedHolder.addValue("nextCount", updatedMetric.getNextCount()); storedHolder.addValue("seekCount", updatedMetric.getSeekCount()); + storedHolder.addValue("evaluatedCount", updatedMetric.getEvaluatedCount()); + storedHolder.addValue("rejectedCount", updatedMetric.getRejectedCount()); storedHolder.addValue("yieldCount", updatedMetric.getYieldCount()); storedHolder.addValue("docRanges", updatedMetric.getDocRanges()); storedHolder.addValue("fiRanges", updatedMetric.getFiRanges()); diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java index d396c23d..6f23c486 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java @@ -94,6 +94,7 @@ public class QueryMetricHandlerProperties { "END_DATE", "ERROR_CODE", "ERROR_MESSAGE", + "EVALUATED_COUNT", "FI_RANGES", "HOST", "LIFECYCLE", @@ -112,6 +113,7 @@ public class QueryMetricHandlerProperties { "QUERY_LOGIC", "QUERY_NAME", "QUERY_TYPE", + "REJECTED_COUNT", "SEEK_COUNT", "SETUP_TIME", "SOURCE_COUNT", @@ -142,9 +144,11 @@ public class QueryMetricHandlerProperties { "CREATE_CALL_TIME", "DOC_RANGES", "ELAPSED_TIME", + "EVALUATED_COUNT", "FI_RANGES", "LOGIN_TIME", "NEXT_COUNT", + "REJECTED_COUNT", "SEEK_COUNT", "SETUP_TIME", "SOURCE_COUNT", diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 27225a72..b0c16a6a 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -215,6 +215,9 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getErrorMessage(), stored == null ? null : stored.getErrorMessage())) { fields.put("ERROR_MESSAGE", updated.getErrorMessage()); } + if (isChanged(updated.getEvaluatedCount(), stored == null ? -1 : stored.getEvaluatedCount())) { + fields.put("EVALUATED_COUNT", Long.toString(updated.getEvaluatedCount())); + } if (isChanged(updated.getFiRanges(), stored == null ? -1 : stored.getFiRanges())) { fields.put("FI_RANGES", Long.toString(updated.getFiRanges())); } @@ -319,6 +322,9 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { fields.put("QUERY_TYPE", updated.getQueryType()); } } + if (isChanged(updated.getRejectedCount(), stored == null ? -1 : stored.getRejectedCount())) { + fields.put("REJECTED_COUNT", Long.toString(updated.getRejectedCount())); + } if (isFirstWrite(updated.getSetupTime(), stored == null ? 0 : stored.getSetupTime(), 0)) { fields.put("SETUP_TIME", Long.toString(updated.getSetupTime())); } @@ -385,6 +391,9 @@ public Multimap getEventFieldsToDelete(T updated, T stored) { if (isChanged(updated.getElapsedTime(), stored.getElapsedTime())) { fields.put("ELAPSED_TIME", Long.toString(stored.getElapsedTime())); } + if (isChanged(updated.getEvaluatedCount(), stored.getEvaluatedCount())) { + fields.put("EVALUATED_COUNT", Long.toString(stored.getEvaluatedCount())); + } if (isChanged(updated.getFiRanges(), stored.getFiRanges())) { fields.put("FI_RANGES", Long.toString(stored.getFiRanges())); } @@ -443,6 +452,9 @@ public Multimap getEventFieldsToDelete(T updated, T stored) { fields.put("PLAN", stored.getPlan()); } } + if (isChanged(updated.getRejectedCount(), stored.getRejectedCount())) { + fields.put("REJECTED_COUNT", Long.toString(stored.getRejectedCount())); + } if (isChanged(updated.getSeekCount(), stored.getSeekCount())) { fields.put("SEEK_COUNT", Long.toString(stored.getSeekCount())); } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 9abe789a..d253f946 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -182,6 +182,8 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setSourceCount(combinedMetric.getSourceCount() + updatedQueryMetric.getSourceCount()); combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount()); combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount()); + combinedMetric.setEvaluatedCount(combinedMetric.getEvaluatedCount() + updatedQueryMetric.getEvaluatedCount()); + combinedMetric.setRejectedCount(combinedMetric.getRejectedCount() + updatedQueryMetric.getRejectedCount()); combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount()); combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges()); combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges()); @@ -189,6 +191,8 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount()); combinedMetric.setNextCount(updatedQueryMetric.getNextCount()); combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount()); + combinedMetric.setEvaluatedCount(updatedQueryMetric.getEvaluatedCount()); + combinedMetric.setRejectedCount(updatedQueryMetric.getRejectedCount()); combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount()); combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges()); combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges()); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 29e6ddce..18a09ea9 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -507,6 +507,15 @@ public T toMetric(EventBase event) { m.setErrorCode(fieldValue); } else if (fieldName.equals("ERROR_MESSAGE")) { m.setErrorMessage(fieldValue); + } else if (fieldName.equals("EVALUATED_COUNT")) { + try { + long l = Long.parseLong(fieldValue); + if (l > m.getEvaluatedCount()) { + m.setEvaluatedCount(l); + } + } catch (Exception e) { + log.error(fieldName + ":" + fieldValue + ":" + e.getMessage()); + } } else if (fieldName.equals("FI_RANGES")) { try { long l = Long.parseLong(fieldValue); @@ -615,6 +624,15 @@ public T toMetric(EventBase event) { m.setQueryName(fieldValue); } else if (fieldName.equals("QUERY_TYPE")) { m.setQueryType(fieldValue); + } else if (fieldName.equals("REJECTED_COUNT")) { + try { + long l = Long.parseLong(fieldValue); + if (l > m.getRejectedCount()) { + m.setRejectedCount(l); + } + } catch (Exception e) { + log.error(fieldName + ":" + fieldValue + ":" + e.getMessage()); + } } else if (fieldName.equals("SEEK_COUNT")) { try { long l = Long.parseLong(fieldValue); diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index 4433b356..0ee051df 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -161,6 +161,8 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception updatedMetric.setSourceCount(queryMetricUpdate.getValue("sourceCount")); updatedMetric.setNextCount(queryMetricUpdate.getValue("nextCount")); updatedMetric.setSeekCount(queryMetricUpdate.getValue("seekCount")); + updatedMetric.setEvaluatedCount(queryMetricUpdate.getValue("evaluatedCount")); + updatedMetric.setRejectedCount(queryMetricUpdate.getValue("rejectedCount")); updatedMetric.setYieldCount(queryMetricUpdate.getValue("yieldCount")); updatedMetric.setDocRanges(queryMetricUpdate.getValue("docRanges")); updatedMetric.setFiRanges(queryMetricUpdate.getValue("fiRanges")); @@ -204,6 +206,8 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception queryMetricUpdate.getMetric().setSourceCount(updatedMetric.getSourceCount()); queryMetricUpdate.getMetric().setNextCount(updatedMetric.getNextCount()); queryMetricUpdate.getMetric().setSeekCount(updatedMetric.getSeekCount()); + queryMetricUpdate.getMetric().setEvaluatedCount(updatedMetric.getEvaluatedCount()); + queryMetricUpdate.getMetric().setRejectedCount(updatedMetric.getRejectedCount()); queryMetricUpdate.getMetric().setYieldCount(updatedMetric.getYieldCount()); queryMetricUpdate.getMetric().setDocRanges(updatedMetric.getDocRanges()); queryMetricUpdate.getMetric().setFiRanges(updatedMetric.getFiRanges()); From 2641af3269d75b49d8f46edcfd95bcd8570e66fc Mon Sep 17 00:00:00 2001 From: foster33 Date: Tue, 21 May 2024 14:52:09 +0000 Subject: [PATCH 2/2] Add counts to metric pages --- .../microservice/querymetric/QueryMetricModel.java | 8 ++++++++ .../microservice/querymetric/QueryMetricModelFormat.java | 4 ++++ .../main/resources/templates/querymetric-horizontal.html | 2 ++ api/src/main/resources/templates/querymetric.html | 2 ++ service/pom.xml | 2 +- 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java index 9ff89414..29679120 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java @@ -111,6 +111,14 @@ public String getSeekCountStr() { return nf.format(seekCount); } + public String getEvaluatedCountStr() { + return nf.format(evaluatedCount); + } + + public String getRejectedCountStr() { + return nf.format(rejectedCount); + } + public String getYieldCountStr() { return nf.format(yieldCount); } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java index d8ec3571..331b9b5d 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java @@ -50,6 +50,10 @@ public interface QueryMetricModelFormat { String getSeekCountStr(); + String getEvaluatedCountStr(); + + String getRejectedCountStr(); + String getYieldCountStr(); String getVersionStr(); diff --git a/api/src/main/resources/templates/querymetric-horizontal.html b/api/src/main/resources/templates/querymetric-horizontal.html index 6286cd63..fa50d011 100644 --- a/api/src/main/resources/templates/querymetric-horizontal.html +++ b/api/src/main/resources/templates/querymetric-horizontal.html @@ -57,6 +57,8 @@

Query Metrics

Sources Next Calls Seek Calls + Evaluated Events + Rejected Events Yield Count Version Total Page Time (ms) diff --git a/api/src/main/resources/templates/querymetric.html b/api/src/main/resources/templates/querymetric.html index bdbc71cb..0b147731 100644 --- a/api/src/main/resources/templates/querymetric.html +++ b/api/src/main/resources/templates/querymetric.html @@ -94,6 +94,8 @@

Query Metrics

Sources Next Calls Seek Calls + Events Evaluated + Events Rejected Yield Count Total Page Time (ms) Total Page Call Time (ms) diff --git a/service/pom.xml b/service/pom.xml index 1bd3f140..1dde14f2 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -28,7 +28,7 @@ 4.0.0 4.0.0 4.0.0 - 4.0.0 + 4.0.1-SNAPSHOT 4.0.0 3.0.0 3.0.0