Merge branch 'main' into async-shard-fetch-psatransport

.github/workflows/check-compatibility.yml

@@ -0,0 +1,35 @@
+---
+name: Check Compatibility
+
+on:
+  pull_request_target
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run compatibility task
+        run: ./gradlew checkCompatibility | tee $HOME/gradlew-check.out
+
+      - name: Get results
+        run: |
+          echo 'Compatibility status:' > ${{ github.workspace }}/results.txt && echo '```' >> ${{ github.workspace }}/results.txt
+          grep -e 'Compatible components' -e 'Incompatible components' -e 'Components skipped' -A 2 -B 3 $HOME/gradlew-check.out >> "${{ github.workspace }}/results.txt"
+          echo '```' >> ${{ github.workspace }}/results.txt
+
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Add comment on the PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ steps.github_app_token.outputs.token }}
+          issue-number: ${{ github.event.number }}
+          body-path: "${{ github.workspace }}/results.txt"

CHANGELOG.md

@@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719))
 - [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792))
 - Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792))
+- [Remote Store] Restrict user override for remote store index level settings ([#8812](https://github.com/opensearch-project/OpenSearch/pull/8812))
 
 ### Deprecated
 
@@ -80,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221)
 - [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195)
 - Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606))
+- Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
@@ -95,6 +97,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.gradle.enterprise` from 3.13.3 to 3.14.1 ([#8996](https://github.com/opensearch-project/OpenSearch/pull/8996))
 - Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 ([#8995](https://github.com/opensearch-project/OpenSearch/pull/8995))
 - Bump `com.google.cloud:google-cloud-core-http` from 2.21.0 to 2.21.1 ([#8999](https://github.com/opensearch-project/OpenSearch/pull/8999))
+- Bump `com.maxmind.geoip2:geoip2` from 4.0.1 to 4.1.0 ([#8998](https://github.com/opensearch-project/OpenSearch/pull/8998))
+- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 in /plugins/repository-hdfs ([#8997](https://github.com/opensearch-project/OpenSearch/pull/8997))
+- Bump `netty` from 4.1.94.Final to 4.1.96.Final ([#9030](https://github.com/opensearch-project/OpenSearch/pull/9030))
 
 ### Changed
 - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303))
@@ -105,7 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807))
 - Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801))
 - [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005))
-
+- Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047))
 ### Deprecated
 
 ### Removed

benchmarks/src/main/java/org/opensearch/benchmark/routing/allocation/Allocators.java

@@ -45,7 +45,7 @@
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;

buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy

@@ -40,8 +40,8 @@ class CheckCompatibilityTask extends DefaultTask {
 
     @TaskAction
     void checkCompatibility() {
-        logger.info("Checking compatibility for: $repositoryUrls for $ref")
         repositoryUrls.parallelStream().forEach { repositoryUrl ->
+            logger.lifecycle("Checking compatibility for: $repositoryUrl with ref: $ref")
             def tempDir = File.createTempDir()
             try {
                 if (cloneAndCheckout(repositoryUrl, tempDir)) {
@@ -81,8 +81,16 @@ class CheckCompatibilityTask extends DefaultTask {
 
     protected static List getRepoUrls() {
         def json = new JsonSlurper().parse(REPO_URL.toURL())
-        def labels = json.projects.values()
-        return labels as List
+        def repository = json.projects.values()
+        def repoUrls = replaceSshWithHttps(repository as List)
+        return repoUrls
+    }
+
+    protected static replaceSshWithHttps(List<String> repoList) {
+        repoList.replaceAll { element ->
+            element.replace("[email protected]:", "https://github.com/")
+        }
+        return repoList
     }
 
     protected boolean cloneAndCheckout(repoUrl, directory) {

buildSrc/version.properties

@@ -28,7 +28,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.94.Final
+netty             = 4.1.96.Final
 joda              = 2.12.2
 
 # client dependencies

client/rest-high-level/src/test/java/org/opensearch/client/tasks/CancelTasksResponseTests.java

@@ -40,7 +40,7 @@
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.common.xcontent.XContentType;

libs/common/src/main/java/org/opensearch/common/network/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** common network layer classes used across the code base */
+package org.opensearch.common.network;

server/src/main/java/org/opensearch/common/recycler/Recyclers.java → libs/common/src/main/java/org/opensearch/common/recycler/Recyclers.java

@@ -75,6 +75,8 @@ public static <T> Recycler.Factory<T> dequeFactory(final Recycler.C<T> c, final
     /**
      * Wrap the provided recycler so that calls to {@link Recycler#obtain()} and {@link Recycler.V#close()} are protected by
      * a lock.
+     *
+     * @opensearch.internal
      */
     public static <T> Recycler<T> locked(final Recycler<T> recycler) {
         return new FilterRecycler<T>() {
@@ -140,7 +142,7 @@ public static <T> Recycler<T> concurrent(final Recycler.Factory<T> factory, fina
             private final Recycler<T>[] recyclers;
 
             {
-                @SuppressWarnings("unchecked")
+                @SuppressWarnings({ "rawtypes", "unchecked" })
                 final Recycler<T>[] recyclers = new Recycler[concurrencyLevel];
                 this.recyclers = recyclers;
                 for (int i = 0; i < concurrencyLevel; ++i) {

libs/common/src/main/java/org/opensearch/common/recycler/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Common Recycler functionality for recycling objects */
+package org.opensearch.common.recycler;

server/src/main/java/org/opensearch/common/transport/package-info.java → libs/common/src/main/java/org/opensearch/common/transport/package-info.java

@@ -6,5 +6,5 @@
  * compatible open source license.
  */
 
-/** Base Transport utility package. */
+/** common transport layer classes used across the code base */
 package org.opensearch.common.transport;

server/src/main/java/org/opensearch/common/transport/BoundTransportAddress.java → libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java

@@ -30,12 +30,12 @@
  * GitHub history for details.
  */
 
-package org.opensearch.common.transport;
+package org.opensearch.core.common.transport;
 
+import org.opensearch.common.network.InetAddresses;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.network.InetAddresses;
 
 import java.io.IOException;
 

server/src/main/java/org/opensearch/common/transport/TransportAddress.java → libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java

@@ -30,12 +30,12 @@
  * GitHub history for details.
  */
 
-package org.opensearch.common.transport;
+package org.opensearch.core.common.transport;
 
+import org.opensearch.common.network.NetworkAddress;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.network.NetworkAddress;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 

libs/core/src/main/java/org/opensearch/core/common/transport/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Common / Base Transport classes used to implement the OpenSearch transport layer */
+package org.opensearch.core.common.transport;

server/src/main/java/org/opensearch/transport/TransportMessage.java → libs/core/src/main/java/org/opensearch/core/transport/TransportMessage.java

@@ -30,11 +30,11 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.core.transport;
 
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 
 /**
  * Message over the transport interface

server/src/main/java/org/opensearch/transport/TransportResponse.java → libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java

@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.core.transport;
 
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;

libs/core/src/main/java/org/opensearch/core/transport/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Core Transport Layer classes used across the OpenSearch core */
+package org.opensearch.core.transport;

modules/ingest-geoip/build.gradle

@@ -39,7 +39,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:4.0.1')
+  api('com.maxmind.geoip2:geoip2:4.1.0')
   // geoip2 dependencies:
   api('com.maxmind.db:maxmind-db:3.0.0')
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")

modules/ingest-geoip/licenses/geoip2-4.1.0.jar.sha1

@@ -0,0 +1 @@
+b6b356cc91863409ba3475a148ee11a3a6d6aa4b

modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java

@@ -254,7 +254,7 @@ protected PercolateQueryBuilder(String field, Supplier<BytesReference> documentS
         }
         documents = in.readList(StreamInput::readBytesReference);
         if (documents.isEmpty() == false) {
-            if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (in.getVersion().onOrAfter(Version.V_2_10_0)) {
                 documentXContentType = in.readMediaType();
             } else {
                 documentXContentType = in.readEnum(XContentType.class);
@@ -304,7 +304,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             out.writeBytesReference(document);
         }
         if (documents.isEmpty() == false) {
-            if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            if (out.getVersion().onOrAfter(Version.V_2_10_0)) {
                 documentXContentType.writeTo(out);
             } else {
                 out.writeEnum((XContentType) documentXContentType);

modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java

@@ -50,7 +50,7 @@
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;

modules/reindex/src/test/java/org/opensearch/index/reindex/RetryTests.java

@@ -42,7 +42,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.http.HttpInfo;
 import org.opensearch.index.query.QueryBuilders;

modules/transport-netty4/licenses/netty-buffer-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+4b80fffbe77485b457bf844289bf1801f61b9e91

modules/transport-netty4/licenses/netty-codec-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+9cfe430f8b14e7ba86969d8e1126aa0aae4d18f0

modules/transport-netty4/licenses/netty-codec-http-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+a4d0d95df5026965c454902ef3d6d84b81f89626

modules/transport-netty4/licenses/netty-codec-http2-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+cc8baf4ff67c1bcc0cde60bc5c2bb9447d92d9e6

modules/transport-netty4/licenses/netty-common-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+d10c167623cbc471753f950846df241d1021655c

modules/transport-netty4/licenses/netty-handler-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+7840d7523d709e02961b647546f9d9dde1699306

modules/transport-netty4/licenses/netty-resolver-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+0e51db5568a881e0f9b013b35617c597dc32f130

modules/transport-netty4/licenses/netty-transport-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+dbd15ca244be28e1a98ed29b9d755edbfa737e02

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.96.Final.jar.sha1

@@ -0,0 +1 @@
+daf8578cade63a01525ee9d70371fa78e6e91094

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4Http2IT.java

@@ -13,7 +13,7 @@
 import io.netty.util.ReferenceCounted;
 import org.opensearch.OpenSearchNetty4IntegTestCase;
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java

@@ -38,7 +38,7 @@
 import org.opensearch.OpenSearchNetty4IntegTestCase;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.http.HttpServerTransport;

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4PipeliningIT.java

@@ -35,7 +35,7 @@
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.util.ReferenceCounted;
 import org.opensearch.OpenSearchNetty4IntegTestCase;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;

modules/transport-netty4/src/internalClusterTest/java/org/opensearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java

@@ -36,8 +36,8 @@
 import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.opensearch.common.network.NetworkAddress;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.BoundTransportAddress;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.BoundTransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;
 import org.opensearch.test.junit.annotations.Network;