enabling cache and 10k leaf

Signed-off-by: Bharathwaj G <[email protected]>
bharath-techie · Feb 20, 2024 · ebb9da1 · ebb9da1
1 parent aab47d1
commit ebb9da1
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 10 deletions.
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -267,7 +267,7 @@ public Iterator<Setting<?>> settings() {
     // whether to use the query cache
     public static final Setting<Boolean> INDEX_QUERY_CACHE_ENABLED_SETTING = Setting.boolSetting(
         "index.queries.cache.enabled",
-        false,
+        true,
         Property.IndexScope
     );
 

diff --git a/...src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/...src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java
@@ -101,6 +101,7 @@ public abstract class BaseSingleTreeBuilder {
         indexOutput = state.directory.createOutput(docFileName, state.context);
         CodecUtil.writeIndexHeader(indexOutput, "STARTreeCodec", 0, state.segmentInfo.getId(), state.segmentSuffix);
         dimensionsSplitOrder = new ArrayList<>();
+
         dimensionsSplitOrder.add("minute");
         dimensionsSplitOrder.add("hour");
         dimensionsSplitOrder.add("day");

diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java
@@ -18,8 +18,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.DocIdSetBuilder;
@@ -51,11 +49,14 @@ static class StarTreeResult {
         final DocIdSetBuilder _matchedDocIds;
         final Set<String> _remainingPredicateColumns;
         final int numOfMatchedDocs;
+        final int maxMatchedDoc;
 
-        StarTreeResult(DocIdSetBuilder matchedDocIds, Set<String> remainingPredicateColumns, int numOfMatchedDocs) {
+        StarTreeResult(DocIdSetBuilder matchedDocIds, Set<String> remainingPredicateColumns, int numOfMatchedDocs,
+            int maxMatchedDoc) {
             _matchedDocIds = matchedDocIds;
             _remainingPredicateColumns = remainingPredicateColumns;
             this.numOfMatchedDocs = numOfMatchedDocs;
+            this.maxMatchedDoc = maxMatchedDoc;
         }
     }
 
@@ -68,7 +69,6 @@ static class StarTreeResult {
 
     DocIdSetBuilder.BulkAdder adder;
     Map<String, SortedNumericDocValues> dimValueMap;
-    int docNum;
     public StarTreeFilter(
         StarTreeAggregatedValues starTreeAggrStructure,
         Map<String, List<Predicate<Long>>> predicateEvaluators,
@@ -82,7 +82,6 @@ public StarTreeFilter(
 
         // TODO : this should be the maximum number of doc values
         docsWithField = new DocIdSetBuilder(Integer.MAX_VALUE);
-        docNum = 0;
     }
 
     /**
@@ -97,13 +96,14 @@ public StarTreeFilter(
     // 1706268600 / (60*60*1000) * (60*60*1000)
     public DocIdSetIterator getStarTreeResult() throws IOException {
         StarTreeResult starTreeResult = traverseStarTree();
+        logger.info("Matched docs in star tree : {}" , starTreeResult.numOfMatchedDocs);
         List<DocIdSetIterator> andIterators = new ArrayList<>();
         andIterators.add(starTreeResult._matchedDocIds.build().iterator());
         DocIdSetIterator docIdSetIterator = andIterators.get(0);
         int docCount = 0;
         for (String remainingPredicateColumn : starTreeResult._remainingPredicateColumns) {
             // TODO : set to max value of doc values
-            DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs);
+            DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.maxMatchedDoc);
             List<Predicate<Long>> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn);
             SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn);
             List<Integer> docIds = new ArrayList<>();
@@ -166,6 +166,9 @@ private StarTreeResult traverseStarTree() throws IOException {
             globalRemainingPredicateColumns = new HashSet<>(remainingPredicateColumns);
         }
 
+        int matchedDocsCountInStarTree = 0;
+        int maxDocNum = -1;
+
         StarTreeNode starTreeNode;
         List<Integer> docIds = new ArrayList<>();
         while ((starTreeNode = queue.poll()) != null) {
@@ -185,7 +188,8 @@ private StarTreeResult traverseStarTree() throws IOException {
             if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) {
                 int docId = starTreeNode.getAggregatedDocId();
                 docIds.add(docId);
-                docNum = docId > docNum ? docId : docNum;
+                matchedDocsCountInStarTree++;
+                maxDocNum = docId > maxDocNum ? docId : maxDocNum;
                 continue;
             }
 
@@ -197,7 +201,8 @@ private StarTreeResult traverseStarTree() throws IOException {
             if (starTreeNode.isLeaf()) {
                 for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) {
                     docIds.add((int)i);
-                    docNum = (int)i > docNum ? (int)i : docNum;
+                    matchedDocsCountInStarTree++;
+                    maxDocNum = (int)i > maxDocNum ? (int)i : maxDocNum;
                 }
                 continue;
             }
@@ -294,7 +299,8 @@ private StarTreeResult traverseStarTree() throws IOException {
         return new StarTreeResult(
             docsWithField,
             globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(),
-            docNum
+            matchedDocsCountInStarTree,
+            maxDocNum
         );
     }
 }