From 6535d7440dbe8ed0f10f8ecec162e5c84ee72b14 Mon Sep 17 00:00:00 2001 From: Moriarty <22225248+apmoriarty@users.noreply.github.com> Date: Thu, 16 Jan 2025 23:49:28 +0000 Subject: [PATCH] Migrate QueryIteratorIT to using field sets for index only and term frequency fields Corrected exceeded value markers, causing several tests to invert their hitor miss assertion Add alternate forms of negations, i.e. (value !~) and !(value =~) Address various warnings, use a static Value instead of creating a new one each time, reuse the KryoDeserializer Add explanation for change in final assertions --- .../query/iterator/QueryIterator.java | 1 + .../visitors/IteratorBuildingVisitor.java | 1 + .../ancestor/AncestorQueryIteratorIT.java | 4 + .../query/iterator/QueryIteratorIT.java | 489 +++++++++++------- .../query/tld/TLDQueryIteratorIT.java | 67 +-- 5 files changed, 330 insertions(+), 232 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java index cc184abbe26..f25d6502727 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java @@ -935,6 +935,7 @@ protected Iterator> getEvaluation(NestedQueryIterator d // get the function we use for the tf functionality. Note we are // getting an additional source deep copy for this function final Iterator>> itrWithContext; + // TODO: this should be dynamic based on the query fields, not a flag passed to the iterator if (this.isTermFrequenciesRequired()) { TermFrequencyConfig tfConfig = new TermFrequencyConfig(); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IteratorBuildingVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IteratorBuildingVisitor.java index fde988e4bf1..e5e0657207a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IteratorBuildingVisitor.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IteratorBuildingVisitor.java @@ -314,6 +314,7 @@ public Object visit(ASTAndNode and, Object data) { // if we are not limiting the lookup, or not allowing term frequency lookup, // or the field is index only but not in the term frequencies, then we must ivarate + // TODO: this logic is wrong, it will allow an ivarator against a TF field if (!limitLookup || !allowTermFrequencyLookup || (indexOnlyFields.contains(identifier) && !termFrequencyFields.contains(identifier))) { if (source instanceof ASTAndNode) { try { diff --git a/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorQueryIteratorIT.java b/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorQueryIteratorIT.java index 7cf3734244f..58827b84b0f 100644 --- a/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorQueryIteratorIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorQueryIteratorIT.java @@ -5,6 +5,7 @@ import org.junit.Before; import datawave.query.iterator.QueryIteratorIT; +import datawave.query.iterator.QueryOptions; /** * AncestorQueryIterator integration tests. Ancestor Query should find any hits event query finds plus its own unique cases @@ -14,6 +15,9 @@ public class AncestorQueryIteratorIT extends QueryIteratorIT { public void setup() throws IOException { super.setup(); iterator = new AncestorQueryIterator(); + + // adding the HIT_LIST option tells the query iterator to use a HitListArithmetic + options.put(QueryOptions.HIT_LIST, "true"); } /** diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java b/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java index 13c007d5cf8..f436a1e41ba 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java @@ -27,10 +27,8 @@ import java.nio.file.Path; import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -52,6 +50,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; @@ -81,8 +81,8 @@ public class QueryIteratorIT extends EasyMockSupport { protected QueryIterator iterator; protected SortedListKeyValueIterator baseIterator; protected Map options; - protected IteratorEnvironment environment; - protected PluginEnvironment penvironment; + protected IteratorEnvironment iterEnv; + protected PluginEnvironment pluginEnv; protected EventDataQueryFilter filter; protected TypeMetadata typeMetadata; @@ -94,6 +94,13 @@ public class QueryIteratorIT extends EasyMockSupport { public Path tempPath; + protected Set indexedFields = Set.of("EVENT_FIELD1", "EVENT_FIELD4", "EVENT_FIELD6", "TF_FIELD0", "TF_FIELD1", "TF_FIELD2", "INDEX_ONLY_FIELD1", + "INDEX_ONLY_FIELD2", "INDEX_ONLY_FIELD3"); + protected Set indexOnlyFields = Set.of("TF_FIELD0", "TF_FIELD1", "TF_FIELD2", "INDEX_ONLY_FIELD1", "INDEX_ONLY_FIELD2", "INDEX_ONLY_FIELD3"); + protected Set termFrequencyFields = Set.of("TF_FIELD0", "TF_FIELD1", "TF_FIELD2"); + + protected static final Value EMPTY_VALUE = new Value(); + @Before public void setup() throws IOException { iterator = new QueryIterator(); @@ -108,8 +115,9 @@ public void setup() throws IOException { options.put(ALLOW_TERM_FREQUENCY_LOOKUP, "true"); // set the indexed fields list - options.put(INDEXED_FIELDS, - "EVENT_FIELD1,EVENT_FIELD4,EVENT_FIELD6,TF_FIELD0,TF_FIELD1,TF_FIELD2,INDEX_ONLY_FIELD1,INDEX_ONLY_FIELD2,INDEX_ONLY_FIELD3"); + options.put(INDEXED_FIELDS, Joiner.on(',').join(indexedFields)); + options.put(INDEX_ONLY_FIELDS, Joiner.on(',').join(indexOnlyFields)); + options.put(TERM_FREQUENCY_FIELDS, Joiner.on(',').join(termFrequencyFields)); // set the unindexed fields list Multimap> nonIndexedQueryFieldsDatatypes = HashMultimap.create(); @@ -126,9 +134,10 @@ public void setup() throws IOException { options.put(SCAN_ID, "000002"); // setup ivarator settings - IvaratorCacheDirConfig config = new IvaratorCacheDirConfig("file://" + tempPath.toAbsolutePath().toString()); + IvaratorCacheDirConfig config = new IvaratorCacheDirConfig("file://" + tempPath.toAbsolutePath()); options.put(IVARATOR_CACHE_DIR_CONFIG, IvaratorCacheDirConfig.toJson(config)); URL hdfsSiteConfig = this.getClass().getResource("/testhadoop.config"); + Preconditions.checkNotNull(hdfsSiteConfig); options.put(HDFS_SITE_CONFIG_URLS, hdfsSiteConfig.toExternalForm()); // query time range @@ -147,11 +156,11 @@ public void setup() throws IOException { typeMetadata.put("INDEX_ONLY_FIELD2", DEFAULT_DATATYPE, "datawave.data.type.LcNoDiacriticsType"); typeMetadata.put("INDEX_ONLY_FIELD3", DEFAULT_DATATYPE, "datawave.data.type.LcNoDiacriticsType"); - environment = createMock(IteratorEnvironment.class); - penvironment = createMock(PluginEnvironment.class); - EasyMock.expect(environment.getConfig()).andReturn(DefaultConfiguration.getInstance()).anyTimes(); - EasyMock.expect(environment.getPluginEnv()).andReturn(penvironment).anyTimes(); - EasyMock.expect(penvironment.getConfiguration()).andReturn(new ConfigurationImpl(DefaultConfiguration.getInstance())).anyTimes(); + iterEnv = createMock(IteratorEnvironment.class); + pluginEnv = createMock(PluginEnvironment.class); + EasyMock.expect(iterEnv.getConfig()).andReturn(DefaultConfiguration.getInstance()).anyTimes(); + EasyMock.expect(iterEnv.getPluginEnv()).andReturn(pluginEnv).anyTimes(); + EasyMock.expect(pluginEnv.getConfiguration()).andReturn(new ConfigurationImpl(DefaultConfiguration.getInstance())).anyTimes(); filter = createMock(EventDataQueryFilter.class); } @@ -168,35 +177,35 @@ private List> addEvent(String row, String dataType, long ev List> listSource = new ArrayList<>(); // indexed - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD1", "a", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD1", "a", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD1", "a", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD1", "a", dataType, uid, eventTime), EMPTY_VALUE)); // unindexed - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD2", "b", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD3", "c", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD2", "b", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD3", "c", dataType, uid, eventTime), EMPTY_VALUE)); // indexed - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD4", "d", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD4", "d", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD4", "d", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD4", "d", dataType, uid, eventTime), EMPTY_VALUE)); // unindexed - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD5", "e", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD5", "e", dataType, uid, eventTime), EMPTY_VALUE)); // indexed - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD6", "f", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD6", "f", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "EVENT_FIELD6", "f", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "EVENT_FIELD6", "f", dataType, uid, eventTime), EMPTY_VALUE)); // add some indexed TF fields - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(DEFAULT_ROW, "TF_FIELD1", "a,, b,,, c,,", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "a b c", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "a", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "b", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "c", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(DEFAULT_ROW, "TF_FIELD1", "a,, b,,, c,,", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "a b c", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "a", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "b", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD1", "c", dataType, uid, eventTime), EMPTY_VALUE)); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD1", "a", dataType, uid, eventTime), getTFValue(0))); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD1", "b", dataType, uid, eventTime), getTFValue(1))); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD1", "c", dataType, uid, eventTime), getTFValue(2))); - listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "TF_FIELD2", ",x, ,y, ,z,", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "x y z", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "x", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "y", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "z", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent(row, "TF_FIELD2", ",x, ,y, ,z,", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "x y z", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "x", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "y", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "TF_FIELD2", "z", dataType, uid, eventTime), EMPTY_VALUE)); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD2", "x", dataType, uid, eventTime), getTFValue(23))); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD2", "y", dataType, uid, eventTime), getTFValue(24))); listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD2", "z", dataType, uid, eventTime), getTFValue(25))); @@ -206,22 +215,18 @@ private List> addEvent(String row, String dataType, long ev listSource.add(new AbstractMap.SimpleEntry<>(getTF(row, "TF_FIELD4", "d", dataType, uid, eventTime), getTFValue(3))); // add some index only field data - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "apple", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "pear", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "orange", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "beef", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "chicken", dataType, uid, eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "pork", dataType, uid, eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "apple", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "pear", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD1", "orange", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "beef", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "chicken", dataType, uid, eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI(row, "INDEX_ONLY_FIELD2", "pork", dataType, uid, eventTime), EMPTY_VALUE)); return listSource; } protected List> configureTestData(long eventTime) { - List> listSource = new ArrayList<>(); - - listSource.addAll(addEvent(eventTime, "123.345.456")); - - return listSource; + return new ArrayList<>(addEvent(eventTime, "123.345.456")); } protected Range getDocumentRange(String uid) { @@ -253,7 +258,7 @@ public void indexOnly_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "INDEX_ONLY_FIELD1 == 'apple'"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -261,7 +266,7 @@ public void indexOnly_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "INDEX_ONLY_FIELD1 == 'apple'"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -302,7 +307,7 @@ public void indexOnly_documentSpecific_hitTerm_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "INDEX_ONLY_FIELD1 == 'apple'"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -311,14 +316,14 @@ public void indexOnly_shardRange_hitTerm_test() throws IOException { // build the seek range for a shard Range seekRange = getShardRange(); String query = "INDEX_ONLY_FIELD1 == 'apple'"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test public void indexOnly_documentSpecific_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - indexOnly_test(seekRange, "INDEX_ONLY_FIELD1 == 'fork'", true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, "INDEX_ONLY_FIELD1 == 'fork'", true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -326,7 +331,7 @@ public void indexOnly_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "INDEX_ONLY_FIELD1 == 'fork'"; - indexOnly_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -334,7 +339,7 @@ public void indexOnly_documentSpecific_secondEvent_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "INDEX_ONLY_FIELD1 == 'apple'"; - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -343,8 +348,8 @@ public void indexOnly_shardRange_secondEvent_test() throws IOException { Range seekRange = getShardRange(); String query = "INDEX_ONLY_FIELD1 == 'apple'"; Map.Entry>> secondEvent = getBaseExpectedEvent("123.345.457"); - secondEvent.getValue().put("INDEX_ONLY_FIELD1", Arrays.asList(new String[] {"apple"})); - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(secondEvent)); + secondEvent.getValue().put("INDEX_ONLY_FIELD1", List.of("apple")); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), List.of(secondEvent)); } @Test @@ -352,7 +357,7 @@ public void indexOnly_trailingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'ap.*'))"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -360,7 +365,7 @@ public void indexOnly_trailingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'ap.*'))"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -368,7 +373,7 @@ public void indexOnly_trailingRegex_documentSpecific_miss_test() throws IOExcept // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'f.*'))"; - indexOnly_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -376,7 +381,7 @@ public void indexOnly_trailingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'f.*'))"; - indexOnly_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -384,7 +389,7 @@ public void indexOnly_trailingRegex_documentSpecific_secondEvent_test() throws I // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'ap.*'))"; - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -393,8 +398,8 @@ public void indexOnly_trailingRegex_shardRange_secondEvent_test() throws IOExcep Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ 'ap.*'))"; Map.Entry>> secondEvent = getBaseExpectedEvent("123.345.457"); - secondEvent.getValue().put("INDEX_ONLY_FIELD1", Arrays.asList(new String[] {"apple"})); - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(secondEvent)); + secondEvent.getValue().put("INDEX_ONLY_FIELD1", List.of("apple")); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), List.of(secondEvent)); } @Test @@ -402,7 +407,7 @@ public void indexOnly_leadingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*le'))"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -410,7 +415,7 @@ public void indexOnly_leadingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*le'))"; - indexOnly_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -418,7 +423,7 @@ public void indexOnly_leadingRegex_documentSpecific_miss_test() throws IOExcepti // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*k'))"; - indexOnly_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -426,7 +431,7 @@ public void indexOnly_leadingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*k'))"; - indexOnly_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -434,7 +439,7 @@ public void indexOnly_leadingRegex_documentSpecific_secondEvent_test() throws IO // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*le'))"; - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -443,8 +448,8 @@ public void indexOnly_leadingRegex_shardRange_secondEvent_test() throws IOExcept Range seekRange = getShardRange(); String query = "((_Value_ = true) && (INDEX_ONLY_FIELD1 =~ '.*le'))"; Map.Entry>> secondEvent = getBaseExpectedEvent("123.345.457"); - secondEvent.getValue().put("INDEX_ONLY_FIELD1", Arrays.asList(new String[] {"apple"})); - indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(secondEvent)); + secondEvent.getValue().put("INDEX_ONLY_FIELD1", List.of("apple")); + indexOnly_test(seekRange, query, false, addEvent(11, "123.345.457"), List.of(secondEvent)); } @Test @@ -452,7 +457,7 @@ public void event_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -460,7 +465,7 @@ public void event_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -469,7 +474,7 @@ public void event_documentSpecific_hitTerm_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -478,7 +483,7 @@ public void event_shardRange_hitTerm_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -486,7 +491,7 @@ public void event_documentSpecific_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 == 'a'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -494,7 +499,7 @@ public void event_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 == 'a'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -502,7 +507,7 @@ public void event_documentSpecific_secondEvent_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -510,7 +515,7 @@ public void event_shardRange_secondEvent_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 == 'b'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test @@ -518,7 +523,7 @@ public void event_trailingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ 'b.*'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -526,7 +531,7 @@ public void event_trailingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ 'b.*'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -534,7 +539,7 @@ public void event_trailingRegex_documentSpecific_miss_test() throws IOException // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ 'a.*'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -542,7 +547,7 @@ public void event_trailingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ 'a.*'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -550,7 +555,7 @@ public void event_trailingRegex_documentSpecific_secondEvent_test() throws IOExc // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ 'b.*'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -558,7 +563,7 @@ public void event_trailingRegex_shardRange_secondEvent_test() throws IOException // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ 'b.*'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test @@ -566,7 +571,7 @@ public void event_leadingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ '.*b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -574,7 +579,7 @@ public void event_leadingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ '.*b'"; - event_test(seekRange, query, false, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -582,7 +587,7 @@ public void event_leadingRegex_documentSpecific_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ '.*a'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -590,7 +595,7 @@ public void event_leadingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ '.*a'"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -598,7 +603,7 @@ public void event_leadingRegex_documentSpecific_secondEvent_test() throws IOExce // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD2 =~ '.*b'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -606,7 +611,7 @@ public void event_leadingRegex_shardRange_secondEvent_test() throws IOException // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD2 =~ '.*b'"; - event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + event_test(seekRange, query, false, null, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test(expected = IOException.class) @@ -617,7 +622,7 @@ public void event_bogusIvaratorCacheDir_test() throws IOException { // setup a bogus ivarator cache dir for the config options.put(IVARATOR_CACHE_DIR_CONFIG, IvaratorCacheDirConfig.toJson(new IvaratorCacheDirConfig("hddfs://bogusPath"))); - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -625,7 +630,7 @@ public void index_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -633,7 +638,7 @@ public void index_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -642,7 +647,7 @@ public void index_documentSpecific_hitTerm_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -651,7 +656,7 @@ public void index_shardRange_hitTerm_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -659,7 +664,7 @@ public void index_documentSpecific_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD4 == 'e'"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -667,7 +672,7 @@ public void index_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD4 == 'e'"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } /** @@ -681,7 +686,7 @@ public void index_documentSpecific_secondEvent_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } /** @@ -695,7 +700,7 @@ public void index_shardRange_secondEvent_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD4 == 'd'"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test @@ -703,7 +708,7 @@ public void index_trailingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'd.*'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -711,7 +716,7 @@ public void index_trailingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'd.*'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -719,7 +724,7 @@ public void index_trailingRegex_documentSpecific_miss_test() throws IOException // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'e.*'))"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -727,7 +732,7 @@ public void index_trailingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'e.*'))"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -735,7 +740,7 @@ public void index_trailingRegex_documentSpecific_secondEvent_test() throws IOExc // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'd.*'))"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -743,7 +748,7 @@ public void index_trailingRegex_shardRange_secondEvent_test() throws IOException // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ 'd.*'))"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test @@ -751,7 +756,7 @@ public void index_leadingRegex_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*d'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -759,7 +764,7 @@ public void index_leadingRegex_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*d'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test @@ -767,7 +772,7 @@ public void index_leadingRegex_documentSpecific_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*e'))"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -775,7 +780,7 @@ public void index_leadingRegex_shardRange_miss_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*e'))"; - index_test(seekRange, query, true, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -783,7 +788,7 @@ public void index_leadingRegex_documentSpecific_secondEvent_test() throws IOExce // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*d'))"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.EMPTY_LIST); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.emptyList()); } @Test @@ -791,7 +796,7 @@ public void index_leadingRegex_shardRange_secondEvent_test() throws IOException // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "((_Value_ = true) && (EVENT_FIELD4 =~ '.*d'))"; - index_test(seekRange, query, false, addEvent(11, "123.345.457"), Arrays.asList(getBaseExpectedEvent("123.345.457"))); + index_test(seekRange, query, false, addEvent(11, "123.345.457"), Collections.singletonList(getBaseExpectedEvent("123.345.457"))); } @Test @@ -799,31 +804,55 @@ public void tf_exceededValue_trailingWildcard_documentSpecific_test() throws IOE // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ 'b.*'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_trailingWildcard_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ 'b.*'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD2 !~ 'b.*'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_negation2_exceededValue_trailingWildcard_documentSpecific_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getDocumentRange("123.345.456"); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ 'b.*'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_trailingWildcard_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ 'y.*'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD2 !~ 'y.*'))"; + // note: correcting this marker caused the query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD2=y} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_index2_exceededValue_trailingWildcard_documentSpecific_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getDocumentRange("123.345.456"); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ 'y.*'))"; + // note: correcting this marker caused the query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD2=y} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_trailingWildcard_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ 'b.*'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ 'b.*'))"; + // note: correcting this marker caused the query to hit + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD1=b} satisfies the query + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -831,31 +860,49 @@ public void tf_exceededValue_trailingWildcard_shardRange_test() throws IOExcepti // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ 'b.*'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_trailingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 !~ 'b.*'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 !~ 'b.*'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_trailingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 !~ 'z.*'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 !~ 'z.*'))"; + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); + } + + @Test + public void tf_negation2_exceededValue_trailingWildcard_shardRange_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getShardRange(); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ 'b.*'))"; + tf_test(seekRange, query, null, Collections.emptyList(), Collections.emptyList()); + } + + @Test + public void tf_index2_exceededValue_trailingWildcard_shardRange_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getShardRange(); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ 'z.*'))"; + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_trailingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ 'b.*'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ 'b.*'))"; + // note: correcting the marker caused this query to hit + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD1=b} satisfies the query + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test @@ -863,23 +910,45 @@ public void tf_exceededValue_leadingWildcard_documentSpecific_test() throws IOEx // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ '.*b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_leadingWildcard_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD2 !~ '.*b'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_leadingWildcard_documentSpecific_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*x'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD2 !~ '.*x'))"; + // note: correcting this marker caused the query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD2=x} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_negation2_exceededValue_leadingWildcard_documentSpecific_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getDocumentRange("123.345.456"); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*b'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_index2_exceededValue_leadingWildcard_documentSpecific_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getDocumentRange("123.345.456"); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*x'))"; + // note: correcting this marker caused the query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD2=x} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test @@ -887,73 +956,97 @@ public void tf_exceededValue_leadingWildcard_shardRange_test() throws IOExceptio // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ '.*b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_leadingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 !~ '.*b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 !~ '.*b'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_leadingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 !~ '.*a'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 !~ '.*a'))"; + // note: correcting the marker caused this query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD1=a} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_negation2_exceededValue_leadingWildcard_shardRange_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getShardRange(); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*b'))"; + tf_test(seekRange, query, null, Collections.emptyList(), Collections.emptyList()); + } + + // alternate form of negation + @Test + public void tf_index2_exceededValue_leadingWildcard_shardRange_test() throws IOException { + // build the seek range for a document specific pull + Range seekRange = getDocumentRange("123.345.456"); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*a'))"; + // note: correcting the marker caused this query to miss + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD1=a} does not satisfy the query + index_test(seekRange, query, true, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_leadingWildcard_shardRange_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ '.*b'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ '.*b'))"; + // note: correcting the marker caused this query to hit + // this behavior is correct because the document {EVENT_FIELD1=a TF_FIELD1=b} satisfies the query + event_test(seekRange, query, false, null, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_exceededValue_negated_leadingWildcard_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_negated_leadingWildcard_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 !~ '.*z'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_negated_leadingWildcard_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*z'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 !~ '.*z'))"; + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_negated_leadingWildcard_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 =~ '.*z'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*z'))"; + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_exceededValue_negated_leadingWildcard_multiTerm_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ '.*b c'))"; - tf_test(seekRange, query, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*b c'))"; + tf_test(seekRange, query, null, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_exceededValue_negated_leadingWildcard_multiTerm_shardRange_test() throws IOException { Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ '.*b c'))"; - tf_test(seekRange, query, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*b c'))"; + tf_test(seekRange, query, null, Collections.emptyList(), Collections.emptyList()); } @@ -961,49 +1054,49 @@ public void tf_exceededValue_negated_leadingWildcard_multiTerm_shardRange_test() public void tf_exceededValue_negated_leadingWildcard_shardRange_test() throws IOException { Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_negated_leadingWildcard_shardRange_test() throws IOException { Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 =~ '.*z'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*z'))"; + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_exceededValue_negated_leadingWildcardMissIndexOnly_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD4 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_negation_exceededValue_negated_leadingWildcardMissIndexOnly_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 !~ '.*z'))"; + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_index_exceededValue_negated_leadingWildcardMissIndexOnly_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 !~ '.*z'))"; - index_test(seekRange, query, false, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 !~ '.*z'))"; + index_test(seekRange, query, false, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_negated_leadingWildcardMissIndexOnly_documentSpecific_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 =~ '.*z'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*z'))"; + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } @Test public void tf_exceededValue_negated_leadingWildcardMissIndexOnly_shardRange_test() throws IOException { Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD4 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } // terms 'a' and 'b' are adjacent, thus a valid phrase @@ -1011,7 +1104,7 @@ public void tf_exceededValue_negated_leadingWildcardMissIndexOnly_shardRange_tes public void tf_contentFunction_validPhrase_shardRange_test() throws IOException { Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && ((TF_FIELD1 =='a' && TF_FIELD1 =='b') && content:phrase(TF_FIELD1,termOffsetMap,'a','b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } // terms 'a' and 'c' do not appear adjacent @@ -1019,7 +1112,7 @@ public void tf_contentFunction_validPhrase_shardRange_test() throws IOException public void tf_contentFunction_invalidPhrase_shardRange_test() throws IOException { Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && ((TF_FIELD1 =='a' && TF_FIELD1 =='c') && content:phrase(TF_FIELD1,termOffsetMap,'a','c'))"; - tf_test(seekRange, query, new AbstractMap.SimpleEntry(null, null), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, new AbstractMap.SimpleEntry<>(null, null), Collections.emptyList(), Collections.emptyList()); } // terms 'a' and 'b' are adjacent, thus a valid phrase @@ -1027,7 +1120,7 @@ public void tf_contentFunction_invalidPhrase_shardRange_test() throws IOExceptio public void tf_contentFunction_validPhrase_docRange_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && ((TF_FIELD1 =='a' && TF_FIELD1 =='b') && content:phrase(TF_FIELD1,termOffsetMap,'a','b'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } // terms 'a' and 'c' do not appear adjacent @@ -1035,7 +1128,7 @@ public void tf_contentFunction_validPhrase_docRange_test() throws IOException { public void tf_contentFunction_invalidPhrase_docRange_test() throws IOException { Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && ((TF_FIELD1 =='a' && TF_FIELD1 =='c') && content:phrase(TF_FIELD1,termOffsetMap,'a','c'))"; - tf_test(seekRange, query, new AbstractMap.SimpleEntry(null, null), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, new AbstractMap.SimpleEntry<>(null, null), Collections.emptyList(), Collections.emptyList()); } // A && (phrase:'a,b' || phrase:'j,k') @@ -1045,7 +1138,7 @@ public void tf_contentFunction_validPhrase_docRange_test2() throws IOException { Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && (((TF_FIELD1 =='a' && TF_FIELD1 =='b') && content:phrase(TF_FIELD1,termOffsetMap,'a','b')) || " + "((TF_FIELD0 =='j' && TF_FIELD0 =='k') && content:phrase(TF_FIELD0,termOffsetMap,'j','k')))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } // A && (phrase:'a,b' || phrase:'j,k') @@ -1055,14 +1148,14 @@ public void tf_contentFunction_validPhrase_shardRange_test3() throws IOException Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && (((TF_FIELD1 =='a' && TF_FIELD1 =='b') && content:phrase(TF_FIELD1,termOffsetMap,'a','b')) || " + "((TF_FIELD0 =='j' && TF_FIELD0 =='k') && content:phrase(TF_FIELD0,termOffsetMap,'j','k')))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.EMPTY_LIST, Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), Collections.emptyList(), Collections.emptyList()); } @Test public void tf_event_exceededValue_negated_leadingWildcardMissIndexOnly_shardRange_test() throws IOException { Range seekRange = getShardRange(); - String query = "EVENT_FIELD1 =='a' && !((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD2 =~ '.*z'))"; - event_test(seekRange, query, true, null, Collections.EMPTY_LIST, Collections.EMPTY_LIST); + String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD2 =~ '.*z'))"; + event_test(seekRange, query, true, null, Collections.emptyList(), Collections.emptyList()); } protected void configureIterator() { @@ -1096,8 +1189,8 @@ protected void event_test(Range seekRange, String query, boolean miss, Map.Entry replayAll(); - iterator.init(baseIterator, options, environment); - iterator.seek(seekRange, Collections.EMPTY_LIST, true); + iterator.init(baseIterator, options, iterEnv); + iterator.seek(seekRange, Collections.emptyList(), true); verifyAll(); @@ -1140,8 +1233,8 @@ protected void index_test(Range seekRange, String query, boolean miss, List>> hit = getBaseExpectedEvent("123.345.456"); - hit.getValue().put("INDEX_ONLY_FIELD1", Arrays.asList(new String[] {"apple"})); + hit.getValue().put("INDEX_ONLY_FIELD1", List.of("apple")); hits.add(hit); } @@ -1230,8 +1323,8 @@ protected void tf_test(Range seekRange, String query, Map.Entry>> getBaseExpectedEvent(String uid) { @@ -1304,16 +1397,16 @@ protected Map.Entry>> getBaseExpectedEvent(String ui protected Map.Entry>> getBaseExpectedEvent(String row, String dataType, String uid) { Key hitKey = new Key(row, dataType + Constants.NULL + uid); Map> expectedDocument = new HashMap<>(); - expectedDocument.put("EVENT_FIELD1", Arrays.asList(new String[] {"a"})); - expectedDocument.put("EVENT_FIELD2", Arrays.asList(new String[] {"b"})); - expectedDocument.put("EVENT_FIELD3", Arrays.asList(new String[] {"c"})); - expectedDocument.put("EVENT_FIELD4", Arrays.asList(new String[] {"d"})); - expectedDocument.put("EVENT_FIELD5", Arrays.asList(new String[] {"e"})); - expectedDocument.put("EVENT_FIELD6", Arrays.asList(new String[] {"f"})); + expectedDocument.put("EVENT_FIELD1", List.of("a")); + expectedDocument.put("EVENT_FIELD2", List.of("b")); + expectedDocument.put("EVENT_FIELD3", List.of("c")); + expectedDocument.put("EVENT_FIELD4", List.of("d")); + expectedDocument.put("EVENT_FIELD5", List.of("e")); + expectedDocument.put("EVENT_FIELD6", List.of("f")); // non-normalized form - expectedDocument.put("TF_FIELD1", Arrays.asList(new String[] {"a,, b,,, c,,"})); - expectedDocument.put("TF_FIELD2", Arrays.asList(new String[] {",x, ,y, ,z,"})); + expectedDocument.put("TF_FIELD1", List.of("a,, b,,, c,,")); + expectedDocument.put("TF_FIELD2", List.of(",x, ,y, ,z,")); return new AbstractMap.SimpleEntry<>(hitKey, expectedDocument); } @@ -1323,9 +1416,7 @@ protected boolean isExpectHitTerm() { } protected void eval(List>>> toEval) throws IOException { - Iterator>>> evalIterator = toEval.iterator(); - while (evalIterator.hasNext()) { - Map.Entry>> evalPair = evalIterator.next(); + for (Map.Entry>> evalPair : toEval) { eval(evalPair.getKey(), evalPair.getValue()); } assertFalse(iterator.hasTop()); @@ -1367,7 +1458,7 @@ protected void eval(Key docKeyHit, Map> docKeys) throws IOEx assertEquals("Unexpected doc size: " + d.getDictionary().size() + "\nGot: " + docSize + "\n" + "expected: " + docKeys, docKeys.keySet().size(), docSize); - // validate the hitlist + // validate the hit list assertEquals("HIT_TERM presence expected: " + isExpectHitTerm() + " actual: " + (d.getDictionary().get(JexlEvaluation.HIT_TERM_FIELD) != null), (d.getDictionary().get(JexlEvaluation.HIT_TERM_FIELD) != null), isExpectHitTerm()); @@ -1392,14 +1483,13 @@ protected void eval(Key docKeyHit, Map> docKeys) throws IOEx // the data should be a set, verify it matches expected Object dictData = d.getDictionary().get(field).getData(); assertNotNull(dictData); - assertTrue("Expected " + expected.size() + " values for '" + field + "' found 1, '" + dictData.toString() + "'\nexpected: " + expected, + assertTrue("Expected " + expected.size() + " values for '" + field + "' found 1, '" + dictData + "'\nexpected: " + expected, dictData instanceof Set); - Set dictSet = (Set) dictData; - assertEquals("Expected " + expected.size() + " values for '" + field + "' found " + dictSet.size() + "\nfound: " + dictSet.toString() - + "\nexpected: " + expected, dictSet.size(), expected.size()); - Iterator dictIterator = dictSet.iterator(); - while (dictIterator.hasNext()) { - String foundString = dictIterator.next().getData().toString(); + Set dictSet = (Set) dictData; + assertEquals("Expected " + expected.size() + " values for '" + field + "' found " + dictSet.size() + "\nfound: " + dictSet + "\nexpected: " + + expected, dictSet.size(), expected.size()); + for (Attribute attribute : (Iterable>) dictSet) { + String foundString = attribute.getData().toString(); assertTrue("could not find " + foundString + " in results! Still had " + expected, expected.remove(foundString)); } // verify that the expected set is now empty @@ -1412,9 +1502,10 @@ protected void eval(Key docKeyHit, Map> docKeys) throws IOEx } } + private final KryoDocumentDeserializer documentDeserializer = new KryoDocumentDeserializer(); + private Map.Entry deserialize(Value value) { - KryoDocumentDeserializer dser = new KryoDocumentDeserializer(); - return dser.apply(new AbstractMap.SimpleEntry(null, value)); + return documentDeserializer.apply(new AbstractMap.SimpleEntry<>(null, value)); } // support methods diff --git a/warehouse/query-core/src/test/java/datawave/query/tld/TLDQueryIteratorIT.java b/warehouse/query-core/src/test/java/datawave/query/tld/TLDQueryIteratorIT.java index cc6753427a6..0dc697d85af 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tld/TLDQueryIteratorIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/tld/TLDQueryIteratorIT.java @@ -34,6 +34,7 @@ public void setup() throws IOException { // update indexed options.put(INDEXED_FIELDS, options.get(INDEXED_FIELDS) + ",TF_FIELD3"); + options.put(TERM_FREQUENCY_FIELDS, options.get(TERM_FREQUENCY_FIELDS) + ",TF_FIELD3"); // update unindexed fields String nonIndexedDataTypes = options.get(NON_INDEXED_DATATYPES); @@ -66,7 +67,7 @@ public void event_isNotNull_documentSpecific_tld_test() throws IOException { tfField1Hits.add("z"); expectedDocument.getValue().put("TF_FIELD3", tfField1Hits); - event_test(seekRange, "EVENT_FIELD2 == 'b' && not(TF_FIELD3 == null)", false, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + event_test(seekRange, "EVENT_FIELD2 == 'b' && not(TF_FIELD3 == null)", false, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } @Test @@ -81,13 +82,13 @@ public void event_isNotNull_shardRange_tld_test() throws IOException { tfField1Hits.add("z"); expectedDocument.getValue().put("TF_FIELD3", tfField1Hits); - event_test(seekRange, "EVENT_FIELD2 == 'b' && not(TF_FIELD3 == null)", false, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + event_test(seekRange, "EVENT_FIELD2 == 'b' && not(TF_FIELD3 == null)", false, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } /** * Document specific range given expected from RangePartitioner and EVENT_FIELD1 index lookup. ExceededValueThreshold TF FIELD triggers TF index lookup for * evaluation. - * + *

* document specific range, tf via TermFrequencyAggregator * * @throws IOException @@ -104,14 +105,14 @@ public void tf_exceededValue_leadingWildcard_documentSpecific_tld_test() throws tfField1Hits.add("r"); expectedDocument.getValue().put("TF_FIELD1", tfField1Hits); - tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } @Test public void tf_exceededValue_leadingWildcard_fullValueMatch_documentSpecific_tld_test() throws IOException { // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); - String query = "EVENT_FIELD1 =='a' && ((ExceededValueThresholdMarkerJexlNode = true) && (TF_FIELD1 =~ '.*r s'))"; + String query = "EVENT_FIELD1 =='a' && ((_Value_ = true) && (TF_FIELD1 =~ '.*r s'))"; Map.Entry>> expectedDocument = getBaseExpectedEvent("123.345.456"); List tfField1Hits = new ArrayList<>(); // from parent @@ -120,12 +121,12 @@ public void tf_exceededValue_leadingWildcard_fullValueMatch_documentSpecific_tld tfField1Hits.add(",,q ,r, ,s,"); expectedDocument.getValue().put("TF_FIELD1", tfField1Hits); - tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } /** * Shard range given expected from RangePartitioner and EVENT_FIELD1 index lookup. ExceededValueThreshold TF FIELD triggers TF index lookup for evaluation. - * + *

* fiFullySatisfies query, Create an ivarator for tf * * @throws IOException @@ -142,7 +143,7 @@ public void tf_exceededValue_leadingWildcard_shardRange_tld_test() throws IOExce tfField1Hits.add("r"); expectedDocument.getValue().put("TF_FIELD1", tfField1Hits); - tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } @Test @@ -150,7 +151,7 @@ public void tf_exceededValue_negated_leadingWildcard_documentSpecific_tld_test() // build the seek range for a document specific pull Range seekRange = getDocumentRange("123.345.456"); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), configureTLDTestData(11), Collections.emptyList()); } @Test @@ -158,7 +159,7 @@ public void tf_exceededValue_negated_leadingWildcard_shardRange_tld_test() throw // build the seek range for a document specific pull Range seekRange = getShardRange(); String query = "EVENT_FIELD1 =='a' && !((_Value_ = true) && (TF_FIELD1 =~ '.*z'))"; - tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, getBaseExpectedEvent("123.345.456"), configureTLDTestData(11), Collections.emptyList()); } @Test @@ -172,7 +173,7 @@ public void tf_exceededValue_trailingWildcard_documentSpecific_test() throws IOE tfField1Hits.add("r"); expectedDocument.getValue().put("TF_FIELD1", tfField1Hits); - tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } @Test @@ -186,7 +187,7 @@ public void tf_exceededValue_trailingWildcard_shardRange_test() throws IOExcepti tfField1Hits.add("r"); expectedDocument.getValue().put("TF_FIELD1", tfField1Hits); - tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.EMPTY_LIST); + tf_test(seekRange, query, expectedDocument, configureTLDTestData(11), Collections.emptyList()); } @Override @@ -194,7 +195,7 @@ protected Range getDocumentRange(String row, String dataType, String uid) { // not a document range if (uid == null) { // get the shard range from the super - return super.getDocumentRange(row, dataType, uid); + return super.getDocumentRange(row, dataType, null); } Key startKey = new Key(row, dataType + Constants.NULL + uid); @@ -206,30 +207,30 @@ private List> configureTLDTestData(long eventTime) { List> listSource = super.configureTestData(eventTime); // add some indexed TF fields in a child - listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD1", ",,q ,r, ,s,", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "q r s", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "q", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "r", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "s", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "q", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "r", "123.345.456.1", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "s", "123.345.456.1", eventTime), new Value())); - - listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD2", ",d, ,e, ,f,", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "d e f", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "d", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "e", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "f", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "d", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "e", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "f", "123.345.456.2", eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD1", ",,q ,r, ,s,", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "q r s", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "q", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "r", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD1", "s", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "q", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "r", "123.345.456.1", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD1", "s", "123.345.456.1", eventTime), EMPTY_VALUE)); + + listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD2", ",d, ,e, ,f,", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "d e f", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "d", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "e", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD2", "f", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "d", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "e", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getTF("TF_FIELD2", "f", "123.345.456.2", eventTime), EMPTY_VALUE)); // add some event data for children - listSource.add(new AbstractMap.SimpleEntry<>(getEvent("EVENT_FIELD7", "1", "123.345.456.1", eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent("EVENT_FIELD7", "1", "123.345.456.1", eventTime), EMPTY_VALUE)); // add some non-event data that is unique for children - listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD3", "z", "123.345.456.2", eventTime), new Value())); - listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD3", "z", "123.345.456.2", eventTime), new Value())); + listSource.add(new AbstractMap.SimpleEntry<>(getEvent("TF_FIELD3", "z", "123.345.456.2", eventTime), EMPTY_VALUE)); + listSource.add(new AbstractMap.SimpleEntry<>(getFI("TF_FIELD3", "z", "123.345.456.2", eventTime), EMPTY_VALUE)); return listSource; }