Merge branch 'opensearch-project:main' into main

.ci/bwcVersions

@@ -19,3 +19,5 @@ BWC_VERSION:
   - "2.7.0"
   - "2.7.1"
   - "2.8.0"
+  - "2.8.1"
+  - "2.9.0"

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604))
+- Implement concurrent aggregations support without profile option ([#7514](https://github.com/opensearch-project/OpenSearch/pull/7514))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -41,6 +42,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
+- Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Add  `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
+- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.83 ([7933](https://github.com/opensearch-project/OpenSearch/pull/7933/))
+- Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928)))
+- Bump `com.azure:azure-storage-blob` from 12.21.1 to 12.22.2 ([7930](https://github.com/opensearch-project/OpenSearch/pull/7930))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -50,6 +58,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
 - Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681))
+- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 
 ### Deprecated
 
@@ -80,18 +91,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Add TokenManager Interface ([#7452](https://github.com/opensearch-project/OpenSearch/pull/7452))
+- Add Remote store as a segment replication source ([#7653](https://github.com/opensearch-project/OpenSearch/pull/7653))
 
 ### Dependencies
+- Bump `jackson` from 2.15.1 to 2.15.2 ([#7897](https://github.com/opensearch-project/OpenSearch/pull/7897))
+- Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
 - Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673))
+- Compress and cache cluster state during validate join request ([#7321](https://github.com/opensearch-project/OpenSearch/pull/7321))
+- [Snapshot Interop] Add Changes in Create Snapshot Flow for remote store interoperability. ([#7118](https://github.com/opensearch-project/OpenSearch/pull/7118))
+- Add new query profile collector fields with concurrent search execution ([#7898](https://github.com/opensearch-project/OpenSearch/pull/7898))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
+-  Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924
 
 ### Security
 

buildSrc/build.gradle

@@ -118,7 +118,9 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.2'
-  api 'com.networknt:json-schema-validator:1.0.81'
+  api 'com.networknt:json-schema-validator:1.0.83'
+  api 'org.jruby.jcodings:jcodings:1.0.58'
+  api 'org.jruby.joni:joni:2.1.48'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"

buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java

@@ -129,13 +129,9 @@ private void setupRepository(Project project, Jdk jdk) {
                 // To distinguish between those, the GA releases have only major version component (fe 17+32),
                 // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
                 // version separator '.' should be enough.
-                artifactPattern = "jdk-"
-                    + jdk.getBaseVersion()
-                    + "+"
-                    + jdk.getBuild()
-                    + "/OpenJDK"
-                    + jdk.getMajor()
-                    + (jdk.getBaseVersion().contains(".") ? "U" : "")
+                artifactPattern = "jdk-" + jdk.getBaseVersion() + "+" + jdk.getBuild() + "/OpenJDK" + jdk.getMajor()
+                // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release
+                    + (jdk.getBaseVersion().contains(".") || jdk.getBaseVersion().matches("^2\\d+$") ? "U" : "")
                     + "-jdk_[classifier]_[module]_hotspot_"
                     + jdk.getBaseVersion()
                     + "_"

buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java

@@ -31,6 +31,8 @@
 
 package org.opensearch.gradle;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
@@ -54,22 +56,26 @@ public static String getLucene() {
         return lucene;
     }
 
-    public static String getBundledJdk(final String platform) {
+    public static String getBundledJdk(final String platform, final String arch) {
         switch (platform) {
             case "darwin": // fall trough
             case "mac":
                 return bundledJdkDarwin;
             case "freebsd":
                 return bundledJdkFreeBSD;
             case "linux":
-                return bundledJdkLinux;
+                return getBundledJdkLinux(arch);
             case "windows":
                 return bundledJdkWindows;
             default:
                 throw new IllegalArgumentException("unknown platform [" + platform + "]");
         }
     }
 
+    public static String getBundledJdk(final String platform) {
+        return getBundledJdk(platform, null);
+    }
+
     public static String getBundledJdkVendor() {
         return bundledJdkVendor;
     }
@@ -84,6 +90,10 @@ public static Map<String, String> getVersions() {
     private static final String bundledJdkFreeBSD;
     private static final String bundledJdkLinux;
     private static final String bundledJdkWindows;
+    private static final String bundledJdkLinux_arm64;
+    private static final String bundledJdkLinux_x64;
+    private static final String bundledJdkLinux_s390x;
+    private static final String bundledJdkLinux_ppc64le;
     private static final String bundledJdkVendor;
     private static final Map<String, String> versions = new HashMap<String, String>();
 
@@ -98,6 +108,12 @@ public static Map<String, String> getVersions() {
         bundledJdkLinux = props.getProperty("bundled_jdk_linux", bundledJdk);
         bundledJdkWindows = props.getProperty("bundled_jdk_windows", bundledJdk);
 
+        // Bundled JDKs per architecture (linux platform)
+        bundledJdkLinux_arm64 = props.getProperty("bundled_jdk_linux_arm64", bundledJdkLinux);
+        bundledJdkLinux_x64 = props.getProperty("bundled_jdk_linux_x64", bundledJdkLinux);
+        bundledJdkLinux_s390x = props.getProperty("bundled_jdk_linux_s390x", bundledJdkLinux);
+        bundledJdkLinux_ppc64le = props.getProperty("bundled_jdk_linux_ppc64le", bundledJdkLinux);
+
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
         }
@@ -119,4 +135,24 @@ private static Properties getVersionProperties() {
     public static boolean isOpenSearchSnapshot() {
         return opensearch.endsWith("-SNAPSHOT");
     }
+
+    private static String getBundledJdkLinux(String arch) {
+        if (StringUtils.isBlank(arch)) {
+            return bundledJdkLinux;
+        }
+
+        switch (arch) {
+            case "aarch64":
+            case "arm64":
+                return bundledJdkLinux_arm64;
+            case "x64":
+                return bundledJdkLinux_x64;
+            case "s390x":
+                return bundledJdkLinux_s390x;
+            case "ppc64le":
+                return bundledJdkLinux_ppc64le;
+            default:
+                throw new IllegalArgumentException("unknown platform [" + arch + "] for 'linux' platform");
+        }
+    }
 }

buildSrc/src/main/java/org/opensearch/gradle/precommit/ValidateJsonAgainstSchemaTask.java

@@ -103,6 +103,7 @@ public void validate(InputChanges inputChanges) throws IOException {
         File jsonSchemaOnDisk = getJsonSchema();
         getLogger().debug("JSON schema : [{}]", jsonSchemaOnDisk.getAbsolutePath());
         SchemaValidatorsConfig config = new SchemaValidatorsConfig();
+        config.setEcma262Validator(true);
         JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7);
         JsonSchema jsonSchema = factory.getSchema(mapper.readTree(jsonSchemaOnDisk), config);
         Map<File, Set<String>> errors = new LinkedHashMap<>();

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.18+10";
+    private static final String SYSTEM_JDK_VERSION = "11.0.19+7";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.6+10";
+    private static final String GRADLE_JDK_VERSION = "17.0.7+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -1,15 +1,16 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0-snapshot-4d1ed9e
+lucene            = 9.7.0-snapshot-204acc3
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 19.0.2+7
-
+bundled_jdk = 20.0.1+9
+# See please https://github.com/adoptium/temurin-build/issues/3371
+bundled_jdk_linux_ppc64le = 20+36
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.15.1
-jackson_databind  = 2.15.1
+jackson           = 2.15.2
+jackson_databind  = 2.15.2
 snakeyaml         = 2.0
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -28,7 +29,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.91.Final
+netty             = 4.1.93.Final
 joda              = 2.12.2
 
 # client dependencies
@@ -41,7 +42,8 @@ commonslogging    = 1.2
 commonscodec      = 1.15
 
 # plugin dependencies
-aws               = 1.12.270
+aws               = 2.20.55
+reactivestreams   = 1.0.4
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
@@ -57,3 +59,6 @@ bytebuddy         = 1.14.3
 
 # benchmark dependencies
 jmh               = 1.35
+
+# compression
+zstd              = 1.5.5-3

client/sniffer/licenses/jackson-core-2.15.2.jar.sha1

@@ -0,0 +1 @@
+a6fe1836469a69b3ff66037c324d75fc66ef137c

distribution/build.gradle

@@ -283,7 +283,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture ->
         "bundled_${platform}_${architecture}" {
           it.platform = platform
-          it.version = VersionProperties.getBundledJdk(platform)
+          it.version = VersionProperties.getBundledJdk(platform, architecture)
           it.vendor = VersionProperties.bundledJdkVendor
           it.architecture = architecture
         }

distribution/tools/plugin-cli/build.gradle

@@ -82,9 +82,6 @@ thirdPartyAudit.ignoreViolations(
 )
 
 thirdPartyAudit.ignoreMissingClasses(
-  'com.github.luben.zstd.BufferPool',
-  'com.github.luben.zstd.ZstdInputStream',
-  'com.github.luben.zstd.ZstdOutputStream',
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',
   'org.objectweb.asm.Attribute',

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.15.2.jar.sha1

@@ -0,0 +1 @@
+4724a65ac8e8d156a24898d50fd5dbd3642870b8

distribution/tools/upgrade-cli/licenses/jackson-databind-2.15.2.jar.sha1

@@ -0,0 +1 @@
+9353b021f10c307c00328f52090de2bdb4b6ff9c

gradle/runtime-jdk-provision.gradle

@@ -32,7 +32,7 @@ if (BuildParams.getIsRuntimeJavaHomeSet()) {
   jdks {
     provisioned_runtime {
       vendor = VersionProperties.bundledJdkVendor
-      version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
+      version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase(), Architecture.current().name().toLowerCase())
       platform = OS.current().name().toLowerCase()
       architecture = Architecture.current().name().toLowerCase()
     }

libs/core/build.gradle

@@ -103,13 +103,23 @@ tasks.named('forbiddenApisMain').configure {
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
     // from log4j
-    'org.osgi.framework.Bundle',
-    'org.osgi.framework.BundleActivator',
-    'org.osgi.framework.BundleContext',
-    'org.osgi.framework.BundleEvent',
-    'org.osgi.framework.SynchronousBundleListener',
-    'org.osgi.framework.wiring.BundleWire',
-    'org.osgi.framework.wiring.BundleWiring'
+    *[
+        'org.osgi.framework.Bundle',
+        'org.osgi.framework.BundleActivator',
+        'org.osgi.framework.BundleContext',
+        'org.osgi.framework.BundleEvent',
+        'org.osgi.framework.SynchronousBundleListener',
+        'org.osgi.framework.wiring.BundleWire',
+        'org.osgi.framework.wiring.BundleWiring',
+    ]  + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
+       'jdk.incubator.vector.ByteVector',
+       'jdk.incubator.vector.FloatVector',
+       'jdk.incubator.vector.IntVector',
+       'jdk.incubator.vector.ShortVector',
+       'jdk.incubator.vector.Vector',
+       'jdk.incubator.vector.VectorOperators',
+       'jdk.incubator.vector.VectorSpecies'
+    ]
   )
 }
 

libs/core/licenses/jackson-core-2.15.2.jar.sha1

@@ -0,0 +1 @@
+a6fe1836469a69b3ff66037c324d75fc66ef137c

libs/core/licenses/lucene-core-9.7.0-snapshot-204acc3.jar.sha1

@@ -0,0 +1 @@
+3aa698cf90f074cbf24acfd7feaaad84c5a6f829