Skip to content

Commit

Permalink
Merge branch 'main' into bucketAggsAssert3
Browse files Browse the repository at this point in the history
  • Loading branch information
@elasticmachine
elasticmachine authored Nov 11, 2023
2 parents fbd9e5f + 5f08929 commit 5af3ca5
Show file tree
Hide file tree
Showing 27 changed files with 1,477 additions and 1,783 deletions.
12 changes: 6 additions & 6 deletions ...ternal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ public void apply(Project project) {
);
}
}
configureCacheability(restIntegTestTask);
});

project.getTasks()
Expand All @@ -127,16 +126,17 @@ public void apply(Project project) {
t.getClusters().forEach(c -> c.plugin(bundle));
}
});
configureCacheability(t);
});
}

private void configureCacheability(RestIntegTestTask restIntegTestTask) {
private void configureCacheability(StandaloneRestIntegTestTask testTask) {
TaskContainer tasks = project.getTasks();
Spec<Task> taskSpec = t -> tasks.withType(StandaloneRestIntegTestTask.class)
.stream()
.filter(task -> task != restIntegTestTask)
.anyMatch(task -> Collections.disjoint(task.getClusters(), restIntegTestTask.getClusters()) == false);
restIntegTestTask.getOutputs()
.filter(task -> task != testTask)
.anyMatch(task -> Collections.disjoint(task.getClusters(), testTask.getClusters()) == false);
testTask.getOutputs()
.doNotCacheIf(
"Caching disabled for this task since it uses a cluster shared by other tasks",
/*
Expand All @@ -147,7 +147,7 @@ private void configureCacheability(RestIntegTestTask restIntegTestTask) {
*/
taskSpec
);
restIntegTestTask.getOutputs().upToDateWhen(new NotSpec(taskSpec));
testTask.getOutputs().upToDateWhen(new NotSpec(taskSpec));
}

private String systemProperty(String propName) {
Expand Down
6 changes: 0 additions & 6 deletions docs/changelog/101599.yaml
View file

This file was deleted.

293 changes: 151 additions & 142 deletions ...common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
View file

Large diffs are not rendered by default.

49 changes: 23 additions & 26 deletions ...data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
Expand Down Expand Up @@ -86,7 +85,6 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.transport.MockTransportService;
Expand Down Expand Up @@ -122,6 +120,7 @@
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.arrayWithSize;
Expand Down Expand Up @@ -1599,21 +1598,22 @@ public void testSegmentsSortedOnTimestampDesc() throws Exception {
indexDocs("metrics-foo", numDocs3); // 3rd segment
int totalDocs = numDocs1 + numDocs2 + numDocs3;

SearchSourceBuilder source = new SearchSourceBuilder();
source.fetchField(new FieldAndFormat(DEFAULT_TIMESTAMP_FIELD, "epoch_millis"));
source.size(totalDocs);
SearchRequest searchRequest = new SearchRequest(new String[] { "metrics-foo" }, source);
SearchResponse searchResponse = client().search(searchRequest).actionGet();
assertEquals(totalDocs, searchResponse.getHits().getTotalHits().value);
SearchHit[] hits = searchResponse.getHits().getHits();
assertEquals(totalDocs, hits.length);

// Test that when we read data, segments come in the reverse order with a segment with the latest date first
long timestamp1 = Long.valueOf(hits[0].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of 1st seg
long timestamp2 = Long.valueOf(hits[0 + numDocs3].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of the 2nd seg
long timestamp3 = Long.valueOf(hits[0 + numDocs3 + numDocs2].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of the 3rd seg
assertTrue(timestamp1 > timestamp2);
assertTrue(timestamp2 > timestamp3);
assertResponse(
prepareSearch("metrics-foo").addFetchField(new FieldAndFormat(DEFAULT_TIMESTAMP_FIELD, "epoch_millis")).setSize(totalDocs),
resp -> {
assertEquals(totalDocs, resp.getHits().getTotalHits().value);
SearchHit[] hits = resp.getHits().getHits();
assertEquals(totalDocs, hits.length);

// Test that when we read data, segments come in the reverse order with a segment with the latest date first
long timestamp1 = Long.valueOf(hits[0].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of 1st seg
long timestamp2 = Long.valueOf(hits[0 + numDocs3].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of the 2nd seg
long timestamp3 = Long.valueOf(hits[0 + numDocs3 + numDocs2].field(DEFAULT_TIMESTAMP_FIELD).getValue()); // 1st doc of the
// 3rd seg
assertTrue(timestamp1 > timestamp2);
assertTrue(timestamp2 > timestamp3);
}
);
}

public void testCreateDataStreamWithSameNameAsIndexAlias() throws Exception {
Expand Down Expand Up @@ -1914,12 +1914,10 @@ static void indexDocs(String dataStream, int numDocs) {
}

static void verifyDocs(String dataStream, long expectedNumHits, List<String> expectedIndices) {
SearchRequest searchRequest = new SearchRequest(dataStream);
searchRequest.source().size((int) expectedNumHits);
SearchResponse searchResponse = client().search(searchRequest).actionGet();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(expectedNumHits));

Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { assertTrue(expectedIndices.contains(hit.getIndex())); });
assertResponse(prepareSearch(dataStream).setSize((int) expectedNumHits), resp -> {
assertThat(resp.getHits().getTotalHits().value, equalTo(expectedNumHits));
Arrays.stream(resp.getHits().getHits()).forEach(hit -> assertTrue(expectedIndices.contains(hit.getIndex())));
});
}

static void verifyDocs(String dataStream, long expectedNumHits, long minGeneration, long maxGeneration) {
Expand Down Expand Up @@ -2076,9 +2074,8 @@ public void testSearchWithRouting() throws IOException, ExecutionException, Inte
).actionGet();
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("my-logs");
client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
SearchRequest searchRequest = new SearchRequest("my-logs").routing("123");
SearchResponse searchResponse = client().search(searchRequest).actionGet();
assertEquals(searchResponse.getTotalShards(), 4);

assertResponse(prepareSearch("my-logs").setRouting("123"), resp -> { assertEquals(resp.getTotalShards(), 4); });
}

public void testWriteIndexWriteLoadAndAvgShardSizeIsStoredAfterRollover() throws Exception {
Expand Down
64 changes: 36 additions & 28 deletions ...-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import org.elasticsearch.action.ingest.SimulateDocumentBaseResult;
import org.elasticsearch.action.ingest.SimulatePipelineRequest;
import org.elasticsearch.action.ingest.SimulatePipelineResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.bytes.BytesReference;
Expand Down Expand Up @@ -69,6 +68,7 @@

import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
Expand Down Expand Up @@ -250,35 +250,43 @@ public void testGeoIpDatabasesDownload() throws Exception {
state.getDatabases().keySet()
);
GeoIpTaskState.Metadata metadata = state.get(id);
BoolQueryBuilder queryBuilder = new BoolQueryBuilder().filter(new MatchQueryBuilder("name", id))
.filter(new RangeQueryBuilder("chunk").from(metadata.firstChunk()).to(metadata.lastChunk(), true));
int size = metadata.lastChunk() - metadata.firstChunk() + 1;
SearchResponse res = prepareSearch(GeoIpDownloader.DATABASES_INDEX).setSize(size)
.setQuery(queryBuilder)
.addSort("chunk", SortOrder.ASC)
.get();
TotalHits totalHits = res.getHits().getTotalHits();
assertEquals(TotalHits.Relation.EQUAL_TO, totalHits.relation);
assertEquals(size, totalHits.value);
assertEquals(size, res.getHits().getHits().length);

List<byte[]> data = new ArrayList<>();

for (SearchHit hit : res.getHits().getHits()) {
data.add((byte[]) hit.getSourceAsMap().get("data"));
}

TarInputStream stream = new TarInputStream(new GZIPInputStream(new MultiByteArrayInputStream(data)));
TarInputStream.TarEntry entry;
while ((entry = stream.getNextEntry()) != null) {
if (entry.name().endsWith(".mmdb")) {
break;
assertResponse(
prepareSearch(GeoIpDownloader.DATABASES_INDEX).setSize(size)
.setQuery(
new BoolQueryBuilder().filter(new MatchQueryBuilder("name", id))
.filter(new RangeQueryBuilder("chunk").from(metadata.firstChunk()).to(metadata.lastChunk(), true))
)
.addSort("chunk", SortOrder.ASC),
res -> {
try {
TotalHits totalHits = res.getHits().getTotalHits();
assertEquals(TotalHits.Relation.EQUAL_TO, totalHits.relation);
assertEquals(size, totalHits.value);
assertEquals(size, res.getHits().getHits().length);

List<byte[]> data = new ArrayList<>();

for (SearchHit hit : res.getHits().getHits()) {
data.add((byte[]) hit.getSourceAsMap().get("data"));
}

TarInputStream stream = new TarInputStream(new GZIPInputStream(new MultiByteArrayInputStream(data)));
TarInputStream.TarEntry entry;
while ((entry = stream.getNextEntry()) != null) {
if (entry.name().endsWith(".mmdb")) {
break;
}
}

Path tempFile = createTempFile();
Files.copy(stream, tempFile, StandardCopyOption.REPLACE_EXISTING);
parseDatabase(tempFile);
} catch (Exception e) {
fail(e);
}
}
}

Path tempFile = createTempFile();
Files.copy(stream, tempFile, StandardCopyOption.REPLACE_EXISTING);
parseDatabase(tempFile);
);
} catch (Exception e) {
throw new AssertionError(e);
}
Expand Down
41 changes: 23 additions & 18 deletions ...alClusterTest/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.Collection;
import java.util.List;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -72,12 +73,12 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
public void testSearchReturnsTokenCount() throws IOException {
init();

assertSearchReturns(searchById("single"), "single");
assertSearchReturns(searchById("bulk1"), "bulk1");
assertSearchReturns(searchById("bulk2"), "bulk2");
assertSearchReturns(searchById("multi"), "multi");
assertSearchReturns(searchById("multibulk1"), "multibulk1");
assertSearchReturns(searchById("multibulk2"), "multibulk2");
assertResponse(searchById("single"), resp -> assertSearchReturns(resp, "single"));
assertResponse(searchById("bulk1"), resp -> assertSearchReturns(resp, "bulk1"));
assertResponse(searchById("bulk2"), resp -> assertSearchReturns(resp, "bulk2"));
assertResponse(searchById("multi"), resp -> assertSearchReturns(resp, "multi"));
assertResponse(searchById("multibulk1"), resp -> assertSearchReturns(resp, "multibulk1"));
assertResponse(searchById("multibulk2"), resp -> assertSearchReturns(resp, "multibulk2"));
}

/**
Expand All @@ -86,11 +87,14 @@ public void testSearchReturnsTokenCount() throws IOException {
public void testSearchByTokenCount() throws IOException {
init();

assertSearchReturns(searchByNumericRange(4, 4).get(), "single");
assertSearchReturns(searchByNumericRange(10, 10).get(), "multibulk2");
assertSearchReturns(searchByNumericRange(7, 10).get(), "multi", "multibulk1", "multibulk2");
assertSearchReturns(searchByNumericRange(1, 10).get(), "single", "bulk1", "bulk2", "multi", "multibulk1", "multibulk2");
assertSearchReturns(searchByNumericRange(12, 12).get());
assertResponse(searchByNumericRange(4, 4), response -> assertSearchReturns(response, "single"));
assertResponse(searchByNumericRange(10, 10), response -> assertSearchReturns(response, "multibulk2"));
assertResponse(searchByNumericRange(7, 10), response -> assertSearchReturns(response, "multi", "multibulk1", "multibulk2"));
assertResponse(
searchByNumericRange(1, 10),
response -> assertSearchReturns(response, "single", "bulk1", "bulk2", "multi", "multibulk1", "multibulk2")
);
assertResponse(searchByNumericRange(12, 12), this::assertSearchReturns);
}

/**
Expand All @@ -100,11 +104,12 @@ public void testFacetByTokenCount() throws IOException {
init();

String facetField = randomFrom(Arrays.asList("foo.token_count", "foo.token_count_unstored", "foo.token_count_with_doc_values"));
SearchResponse result = searchByNumericRange(1, 10).addAggregation(AggregationBuilders.terms("facet").field(facetField)).get();
assertSearchReturns(result, "single", "bulk1", "bulk2", "multi", "multibulk1", "multibulk2");
assertThat(result.getAggregations().asList().size(), equalTo(1));
Terms terms = (Terms) result.getAggregations().asList().get(0);
assertThat(terms.getBuckets().size(), equalTo(9));
assertResponse(searchByNumericRange(1, 10).addAggregation(AggregationBuilders.terms("facet").field(facetField)), result -> {
assertSearchReturns(result, "single", "bulk1", "bulk2", "multi", "multibulk1", "multibulk2");
assertThat(result.getAggregations().asList().size(), equalTo(1));
Terms terms = (Terms) result.getAggregations().asList().get(0);
assertThat(terms.getBuckets().size(), equalTo(9));
});
}

private void init() throws IOException {
Expand Down Expand Up @@ -174,8 +179,8 @@ private IndexRequestBuilder prepareIndex(String id, String... texts) throws IOEx
return client().prepareIndex("test").setId(id).setSource("foo", texts);
}

private SearchResponse searchById(String id) {
return prepareTokenCountFieldMapperSearch().setQuery(QueryBuilders.termQuery("_id", id)).get();
private SearchRequestBuilder searchById(String id) {
return prepareTokenCountFieldMapperSearch().setQuery(QueryBuilders.termQuery("_id", id));
}

private SearchRequestBuilder searchByNumericRange(int low, int high) {
Expand Down
Loading

0 comments on commit 5af3ca5

Please sign in to comment.