Merge branch 'opensearch-project:main' into createpit

.ci/dockerOnLinuxExclusions

@@ -12,4 +12,4 @@ ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
-sles-15.2
+sles-15.2

.ci/documentation/issue.md

@@ -0,0 +1,11 @@
+**Is your feature request related to a problem?**
+A new feature has been added.
+
+**What solution would you like?**
+Document the usage of the new feature.
+
+**What alternatives have you considered?**
+N/A
+
+**Do you have any additional context?**
+See please 

.github/CODEOWNERS

@@ -1,3 +1,3 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core
+*   @opensearch-project/opensearch-core @reta
 

.github/workflows/code-hygiene.yml

@@ -0,0 +1,14 @@
+name: Code Hygiene
+
+on: [push, pull_request]
+
+jobs:
+  linelint:
+    runs-on: ubuntu-latest
+    name: Check if all files end in newline
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Linelint
+        uses: fernandrone/[email protected]

.github/workflows/create-documentation-issue.yml

@@ -0,0 +1,41 @@
+name: Create Documentation Issue
+on:
+  pull_request:
+    types:
+      - labeled
+env:
+  PR_NUMBER: ${{ github.event.number }}
+
+jobs:
+  create-issue:
+    if: ${{ github.event.label.name == 'needs-documentation' }}
+    runs-on: ubuntu-latest
+    name: Create Documentation Issue
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Edit the issue template
+        run: |
+          echo "https://github.com/opensearch-project/OpenSearch/pull/${{ env.PR_NUMBER }}." >> ./ci/documentation/issue.md
+          
+      - name: Create Issue From File
+        id: create-issue
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: Add documentation related to new feature
+          content-filepath: ./ci/documentation/issue.md
+          labels: documentation
+          repository: opensearch-project/documentation-website
+          token: ${{ steps.github_app_token.outputs.token }}
+
+      - name: Print Issue
+        run: echo Created related documentation issue ${{ steps.create-issue.outputs.issue-number }}

.github/workflows/delete_backport_branch.yml

@@ -12,4 +12,4 @@ jobs:
       - name: Delete merged branch
         uses: SvanBoxel/delete-merged-branch@main
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/links.yml

@@ -16,4 +16,4 @@ jobs:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/lucene-snapshots.yml

@@ -0,0 +1,55 @@
+# This workflow will check out, build, and publish snapshots of lucene.
+
+name: OpenSearch Lucene snapshots
+
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    inputs:
+      ref:
+        description:
+        required: false
+        default: 'main'
+
+jobs:
+  publish-snapshots:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+
+      - name: Checkout Lucene
+        uses: actions/checkout@v2
+        with:
+          repository: 'apache/lucene'
+          path: lucene
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Set hash
+        working-directory: ./lucene
+        run: |
+          echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
+        id: version
+
+      - name: Publish Lucene to local maven repo.
+        working-directory: ./lucene
+        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
+          aws-region: us-west-2
+
+      - name: Copy files to S3 with the aws CLI.
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ secrets.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress

.github/workflows/wrapper.yml

@@ -7,4 +7,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/wrapper-validation-action@v1

.gitignore

@@ -56,3 +56,6 @@ testfixtures_shared/
 
 # These are generated from .ci/jobs.t
 .ci/jobs/
+
+# build files generated
+doc-tools/missing-doclet/bin/

.linelint.yml

@@ -0,0 +1,48 @@
+# 'true' will fix files
+autofix: true
+
+ignore:
+  - .git/
+  - .gradle/
+  - .idea/
+  - '*.sha1'
+  - '*.txt'
+  - '.github/CODEOWNERS'
+  - 'buildSrc/src/testKit/opensearch.build/LICENSE'
+  - 'buildSrc/src/testKit/opensearch.build/NOTICE'
+  - 'server/licenses/apache-log4j-extras-DEPENDENCIES'
+  # Empty files
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-tar/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/bwc/minor/build.gradle'
+  - 'buildSrc/src/main/resources/buildSrc.marker'
+  - 'buildSrc/src/testKit/opensearch-build-resources/settings.gradle'
+  - 'buildSrc/src/testKit/opensearch.build/settings.gradle'
+  - 'buildSrc/src/testKit/reaper/settings.gradle'
+  - 'buildSrc/src/testKit/symbolic-link-preserving-tar/settings.gradle'
+  - 'buildSrc/src/testKit/testingConventions/empty_test_task/.gitignore'
+  - 'client/rest-high-level/src/main/resources/META-INF/services/org.opensearch.plugins.spi.NamedXContentProvider'
+  - 'distribution/bwc/bugfix/build.gradle'
+  - 'distribution/bwc/maintenance/build.gradle'
+  - 'distribution/bwc/minor/build.gradle'
+  - 'distribution/bwc/staged/build.gradle'
+  - 'libs/ssl-config/src/test/resources/certs/pem-utils/empty.pem'
+  - 'qa/evil-tests/src/test/resources/org/opensearch/common/logging/does_not_exist/nothing_to_see_here'
+  - 'qa/os/centos-6/build.gradle'
+  - 'qa/os/debian-8/build.gradle'
+  - 'qa/os/oel-6/build.gradle'
+  - 'qa/os/oel-7/build.gradle'
+  - 'qa/os/sles-12/build.gradle'
+  # Test requires no new line for these files
+  - 'server/src/test/resources/org/opensearch/action/bulk/simple-bulk11.json'
+  - 'server/src/test/resources/org/opensearch/action/search/simple-msearch5.json'
+
+rules:
+  # checks if file ends in a newline character
+  end-of-file:
+    # set to true to enable this rule
+    enable: true
+
+    # if true also checks if file ends in a single newline character
+    single-new-line: true

DEVELOPER_GUIDE.md

@@ -48,6 +48,8 @@
     - [Distributed Framework](#distributed-framework)
   - [Submitting Changes](#submitting-changes)
   - [Backports](#backports)
+  - [LineLint](#linelint)
+  - [Lucene Snapshots](#lucene-snapshots)
 
 # Developer Guide
 
@@ -472,3 +474,23 @@ See [CONTRIBUTING](CONTRIBUTING.md).
 ## Backports
 
 The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
+
+## LineLint
+A linter in [`code-hygiene.yml`](.github/workflows/code-hygiene.yml) that validates simple newline and whitespace rules in all sorts of files. It can:
+- Recursively check a directory tree for files that do not end in a newline
+- Automatically fix these files by adding a newline or trimming extra newlines.
+
+Rules are defined in `.linelint.yml`.
+
+Executing the binary will automatically search the local directory tree for linting errors.
+
+    linelint .
+
+Pass a list of files or directories to limit your search.
+
+    linelint README.md LICENSE
+
+# Lucene Snapshots
+The Github workflow in [lucene-snapshots.yml](.github/workflows/lucene-snapshots.yml) is a Github worfklow executable by maintainers to build a top-down snapshot build of lucene.
+These snapshots are available to test compatibility with upcoming changes to Lucene by updating the version at [version.properties](buildsrc/version.properties) with the `version-snapshot-sha` version.
+Example: `lucene = 10.0.0-snapshot-2e941fc`.

MAINTAINERS.md

@@ -3,27 +3,27 @@
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
 | Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
+| Anas Alkouz | [anasalkouz](https://github.com/anasalkouz) | Amazon |
+| Andrew Ross   | [andrross](https://github.com/andrross)| Amazon |
+| Andriy Redko | [reta](https://github.com/reta) | Aiven |
 | Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon |
+| Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock) | Amazon |
+| Gopala Krishna Ambareesh | [krishna-ggk](https://github.com/krishna-ggk) |Amazon |
 | Himanshu Setia | [setiah](https://github.com/setiah) | Amazon |
+| Itiyama Sadana | [itiyamas](https://github.com/itiyamas) | Amazon |
+| Kartik Ganesh | [kartg](https://github.com/kartg) | Amazon |
+| Marc Handalian | [mch2](https://github.com/mch2) | Amazon |
+| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon |
 | Nick Knize | [nknize](https://github.com/nknize) | Amazon |
+| Owais Kazi | [owaiskazi19](https://github.com/owaiskazi19) | Amazon |
 | Rabi Panda | [adnapibar](https://github.com/adnapibar) | Amazon |
+| Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon|
+| Ryan Bogan | [ryanbogan](https://github.com/ryanbogan) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
-| Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
-| Gopala Krishna Ambareesh | [krishna-ggk](https://github.com/krishna-ggk) |Amazon |
-| Vengadanathan Srinivasan | [vengadanathan-s](https://github.com/vengadanathan-s) | Amazon |
 | Shweta Thareja |[shwetathareja](https://github.com/shwetathareja) | Amazon |
-| Itiyama Sadana | [itiyamas](https://github.com/itiyamas) | Amazon |
-| Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock) | Amazon |
-| Andrew Ross   | [andross](https://github.com/andrross)| Amazon |
+| Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
 | Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon |
-| Anas Alkouz | [anasalkouz](https://github.com/anasalkouz) | Amazon |
-| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon |
-| Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon|
+| Vengadanathan Srinivasan | [vengadanathan-s](https://github.com/vengadanathan-s) | Amazon |
 | Xue Zhou | [xuezhou25](https://github.com/xuezhou25) | Amazon |
-| Kartik Ganesh | [kartg](https://github.com/kartg) | Amazon |
-| Marc Handalian | [mch2](https://github.com/mch2) | Amazon |
-| Ryan Bogan | [ryanbogan](https://github.com/ryanbogan) | Amazon |
-| Owais Kazi | [owaiskazi19](https://github.com/owaiskazi19) | Amazon |
-
 
 [This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).

README.md

@@ -45,4 +45,4 @@ Copyright OpenSearch Contributors. See [NOTICE](NOTICE.txt) for details.
 
 OpenSearch is a registered trademark of Amazon Web Services.
 
-OpenSearch includes certain Apache-licensed Elasticsearch code from Elasticsearch B.V. and other source code. Elasticsearch B.V. is not the source of that other source code. ELASTICSEARCH is a registered trademark of Elasticsearch B.V.
+OpenSearch includes certain Apache-licensed Elasticsearch code from Elasticsearch B.V. and other source code. Elasticsearch B.V. is not the source of that other source code. ELASTICSEARCH is a registered trademark of Elasticsearch B.V.

RELEASING.md

@@ -1,3 +1,3 @@
 ## Releasing
 
-This project follows [OpenSearch project branching, labelling, and releasing](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).
+This project follows [OpenSearch project branching, labelling, and releasing](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).

SECURITY.md

@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [email protected]. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [email protected]. Please do **not** create a public GitHub issue.

build.gradle

@@ -49,7 +49,9 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.4.2" apply false
-  id "org.gradle.test-retry" version "1.3.1" apply false
+  id "org.gradle.test-retry" version "1.3.2" apply false
+  id "test-report-aggregation"
+  id 'jacoco-report-aggregation'
 }
 
 apply from: 'gradle/build-complete.gradle'
@@ -385,6 +387,15 @@ gradle.projectsEvaluated {
       }
     }
   }
+
+  dependencies {
+    subprojects.findAll { it.pluginManager.hasPlugin('java') }.forEach {
+      testReportAggregation it
+    }
+    subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach {
+     jacocoAggregation it
+    }
+  }
 }
 
 // test retry configuration
@@ -402,6 +413,7 @@ subprojects {
 // eclipse configuration
 allprojects {
   apply plugin: 'eclipse'
+
   // Name all the non-root projects after their path so that paths get grouped together when imported into eclipse.
   if (path != ':') {
     eclipse.project.name = path
@@ -558,3 +570,15 @@ subprojects {
     }
   }
 }
+
+reporting {
+  reports {
+    testAggregateTestReport(AggregateTestReport) { 
+      testType = TestSuiteType.UNIT_TEST
+    }
+  }
+}
+
+tasks.named(JavaBasePlugin.CHECK_TASK_NAME) {
+  dependsOn tasks.named('testAggregateTestReport', TestReport) 
+}

buildSrc/build.gradle

@@ -110,7 +110,7 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:11.3.3'
   api 'org.apache.rat:apache-rat:0.13'
   api 'commons-io:commons-io:2.7'
-  api "net.java.dev.jna:jna:5.10.0"
+  api "net.java.dev.jna:jna:5.11.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'de.thetaphi:forbiddenapis:3.3'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'

buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java

@@ -92,10 +92,9 @@ public static void configureRepositories(Project project) {
                 throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
             }
             String revision = matcher.group(1);
-            // TODO(cleanup) - Setup own lucene snapshot repo
             MavenArtifactRepository luceneRepo = repos.maven(repo -> {
                 repo.setName("lucene-snapshots");
-                repo.setUrl("https://artifacts.opensearch.org/snapshots/lucene/");
+                repo.setUrl("https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/");
             });
             repos.exclusiveContent(exclusiveRepo -> {
                 exclusiveRepo.filter(

buildSrc/src/main/resources/eclipse.settings/org.eclipse.core.resources.prefs

@@ -3,4 +3,4 @@ encoding//src/main/java=UTF-8
 encoding//src/main/resources=UTF-8
 encoding//src/test/java=UTF-8
 encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
+encoding/<project>=UTF-8

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-7.4.1
+7.4.1

buildSrc/src/testKit/testingConventions/build.gradle

@@ -88,6 +88,3 @@ project(':valid_setup_with_base') {
     }
   }
 }
-
-
-