CI-workflow.yml

name: Build and Test ml-commons
# This workflow is triggered on pull requests and push to any branches
on:
  push:
    branches-ignore:
      - 'backport/**'
      - 'create-pull-request/**'
      - 'dependabot/**'
  pull_request_target:
    types: [opened, synchronize, reopened]

permissions:
  id-token: write
  contents: read

jobs:
  Get-Require-Approval:
    uses: ./.github/workflows/require-approval.yml

  Get-CI-Image-Tag:
    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
    with:
      product: opensearch

  Build-ml-linux:
    needs: [Get-Require-Approval, Get-CI-Image-Tag]
    strategy:
      matrix:
        java: [11, 17, 21]
    env:
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

    name: Build and Test MLCommons Plugin on linux
    if: github.repository == 'opensearch-project/ml-commons'
    environment: ${{ needs.Get-Require-Approval.outputs.is-require-approval }}
    outputs:
      build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
    runs-on: ubuntu-latest
    container:
      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
      # this image tag is subject to change as more dependencies and updates will arrive over time
      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
      # need to switch to root so that github actions can install runner binary on container without permission issues.
      options: --user root

    steps:
      - name: Setup Java ${{ matrix.java }}
        uses: actions/setup-java@v1
        with:
          java-version: ${{ matrix.java }}

      - uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.ML_ROLE }}
          aws-region: us-west-2

      - name: Checkout MLCommons
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Build and Run Tests
        id: step-build-test-linux
        run: |
          chown -R 1000:1000 `pwd`
          su `id -un 1000` -c 'whoami && java -version && 
                               export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` &&
                               export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` &&
                               echo "::add-mask::$OPENAI_KEY" &&
                               echo "::add-mask::$COHERE_KEY" &&
                               echo "build and run tests" && ./gradlew build &&
                               echo "Publish to Maven Local" && ./gradlew publishToMavenLocal &&
                               echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3'
          plugin=`basename $(ls plugin/build/distributions/*.zip)`
          echo $plugin
          mv -v plugin/build/distributions/$plugin ./
          echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT

      - name: Upload Coverage Report
        uses: codecov/codecov-action@v1
        with:
          flags: ml-commons
          token: ${{ secrets.CODECOV_TOKEN }}

      - uses: actions/upload-artifact@v4
        with:
          name: ml-plugin-linux-${{ matrix.java }}
          path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
          if-no-files-found: error


  Test-ml-linux-docker:
    needs: [Get-Require-Approval, Build-ml-linux]
    strategy:
      matrix:
        java: [11, 17, 21]

    name: Test MLCommons Plugin on linux docker
    if: github.repository == 'opensearch-project/ml-commons'
    environment: ${{ needs.Get-Require-Approval.outputs.is-require-approval }}
    runs-on: ubuntu-latest

    steps:
      - uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.ML_ROLE }}
          aws-region: us-west-2

      - name: Checkout MLCommons
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: actions/download-artifact@v4.1.7
        with:
          name: ml-plugin-linux-${{ matrix.java }}

      - name: Pull and Run Docker
        run: |
          plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }}
          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`

          if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
              qualifier=-${qualifier}
          else
              qualifier=""
          fi

          docker_version=$version$qualifier

          echo plugin version plugin_version qualifier docker_version
          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"

          pwd && ls -l ./$plugin

          if docker pull opensearchstaging/opensearch:$docker_version
          then
            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-skills ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-skills; fi" >> Dockerfile
            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile
            echo "COPY $plugin /tmp/" >> Dockerfile
            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
            docker build -t opensearch-ml:test .
            echo "imagePresent=true" >> $GITHUB_ENV
          else
            echo "imagePresent=false" >> $GITHUB_ENV
          fi
      - name: Generate Password For Admin
        id: genpass
        run: |
            PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-')
            echo "password={$PASSWORD}" >> $GITHUB_OUTPUT
      - name: Run Docker Image
        if: env.imagePresent == 'true'
        run: |
          cd ..
          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ steps.genpass.outputs.password }} opensearch-ml:test
          sleep 90
      - name: Run MLCommons Test
        if: env.imagePresent == 'true'
        run: |
          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:${{ steps.genpass.outputs.password }} --insecure |grep opensearch-security|wc -l`
          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
          echo "::add-mask::$OPENAI_KEY"
          echo "::add-mask::$COHERE_KEY"
          if [ $security -gt 0 ]
          then
            echo "Security plugin is available"
            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=${{ steps.genpass.outputs.password }}
          else
            echo "Security plugin is NOT available"
            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
          fi

      - name: Upload Coverage Report
        uses: codecov/codecov-action@v3
        with:
          flags: ml-commons
          token: ${{ secrets.CODECOV_TOKEN }}

  Build-ml-windows:
    strategy:
      matrix:
        java: [11, 17, 21]
    name: Build and Test MLCommons Plugin on Windows
    if: github.repository == 'opensearch-project/ml-commons'
    needs: [Get-Require-Approval]
    environment: ${{ needs.Get-Require-Approval.outputs.is-require-approval }}
    runs-on: windows-latest

    steps:
      - name: Setup Java ${{ matrix.java }}
        uses: actions/setup-java@v1
        with:
          java-version: ${{ matrix.java }}

      - uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.ML_ROLE }}
          aws-region: us-west-2

      # ml-commons
      - name: Checkout MLCommons
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Build and Run Tests
        shell: bash
        run: |
          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
          echo "::add-mask::$OPENAI_KEY"
          echo "::add-mask::$COHERE_KEY"
          ./gradlew.bat build
      - name: Publish to Maven Local
        run: |
          ./gradlew publishToMavenLocal
#      - name: Multi Nodes Integration Testing
#        shell: bash
#        run: |
#          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
#          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
#          echo "::add-mask::$OPENAI_KEY"
#          echo "::add-mask::$COHERE_KEY"
#          ./gradlew integTest -PnumNodes=3