From 906eb38cc2895e696eb86e8e8bb3ecdd0ebf42ee Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 13 Jul 2023 14:16:43 -0700 Subject: [PATCH 01/11] update script Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 75e09a7eb..7a47df479 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - pulish-snapshot jobs: build-and-publish-snapshots: @@ -35,7 +36,7 @@ jobs: path: 'build' - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v1.7.0 with: role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} aws-region: us-east-1 From 3c60fc2810b8f7d4ee05f45de242d9208bf80cc3 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 18 Jul 2023 16:09:33 -0700 Subject: [PATCH 02/11] change package name to opensearch-spark Signed-off-by: Peng Huo --- build.sbt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 8414da3db..a1ac8bf5a 100644 --- a/build.sbt +++ b/build.sbt @@ -43,7 +43,7 @@ lazy val commonSettings = Seq( lazy val root = (project in file(".")) .aggregate(flintCore, flintSparkIntegration) .disablePlugins(AssemblyPlugin) - .settings(name := "flint") + .settings(name := "flint", publish / skip := true) lazy val flintCore = (project in file("flint-core")) .disablePlugins(AssemblyPlugin) @@ -55,14 +55,15 @@ lazy val flintCore = (project in file("flint-core")) "org.opensearch.client" % "opensearch-rest-high-level-client" % opensearchVersion exclude ("org.apache.logging.log4j", "log4j-api"), "com.amazonaws" % "aws-java-sdk" % "1.12.397" % "provided" - exclude ("com.fasterxml.jackson.core", "jackson-databind"))) + exclude ("com.fasterxml.jackson.core", "jackson-databind")), + publish / skip := true) lazy val flintSparkIntegration = (project in file("flint-spark-integration")) .dependsOn(flintCore) .enablePlugins(AssemblyPlugin, Antlr4Plugin) .settings( commonSettings, - name := "flint-spark", + name := "opensearch-spark", scalaVersion := scala212, libraryDependencies ++= Seq( "com.amazonaws" % "aws-java-sdk" % "1.12.397" % "provided" From 25ec92bca6dd038c9559ce36bd1dc00802f4d243 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 18 Jul 2023 17:41:06 -0700 Subject: [PATCH 03/11] add sha512 and sha256 Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 7a47df479..5c0aa348f 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -41,6 +41,11 @@ jobs: role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} aws-region: us-east-1 + - name: generate sha and md5 + run: | + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" >> "$i.sha512"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" >> "$i.sha256"; done + - name: Get credentials and publish snapshots to maven run: | export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) From fc8bdee4ef69632e8f3d24e179eaa6ce059cf356 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 18 Jul 2023 19:12:58 -0700 Subject: [PATCH 04/11] update Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 5c0aa348f..df836775a 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -44,7 +44,9 @@ jobs: - name: generate sha and md5 run: | for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" >> "$i.sha512"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha512sum "$i" >> "$i.sha512"; done for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" >> "$i.sha256"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha256sum "$i" >> "$i.sha256"; done - name: Get credentials and publish snapshots to maven run: | From 58be396db2c675fe3638ae65fcec996e53e35c54 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 24 Jul 2023 13:41:10 -0700 Subject: [PATCH 05/11] update opensearch-spark-standalone_2.12-0.1.0-SNAPSHOT.jar Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 2 +- build.sbt | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index df836775a..9f22632d2 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -28,7 +28,7 @@ jobs: java-version: 11 - name: Publish to Local Maven - run: sbt publishM2 + run: sbt standaloneCosmetic/publishM2 - uses: actions/checkout@v3 with: diff --git a/build.sbt b/build.sbt index a1ac8bf5a..ac640cb2b 100644 --- a/build.sbt +++ b/build.sbt @@ -63,12 +63,12 @@ lazy val flintSparkIntegration = (project in file("flint-spark-integration")) .enablePlugins(AssemblyPlugin, Antlr4Plugin) .settings( commonSettings, - name := "opensearch-spark", + name := "flint-spark-integration", scalaVersion := scala212, libraryDependencies ++= Seq( "com.amazonaws" % "aws-java-sdk" % "1.12.397" % "provided" exclude ("com.fasterxml.jackson.core", "jackson-databind"), - "org.scalactic" %% "scalactic" % "3.2.15", + "org.scalactic" %% "scalactic" % "3.2.15" % "test", "org.scalatest" %% "scalatest" % "3.2.15" % "test", "org.scalatest" %% "scalatest-flatspec" % "3.2.15" % "test", "org.scalatestplus" %% "mockito-4-6" % "3.2.15.0" % "test", @@ -112,3 +112,23 @@ lazy val integtest = (project in file("integ-test")) "org.testcontainers" % "testcontainers" % "1.18.0" % "test"), libraryDependencies ++= deps(sparkVersion), Test / fullClasspath += (flintSparkIntegration / assembly).value) + +lazy val standaloneCosmetic = project + .settings( + name := "opensearch-spark-standalone", + commonSettings, + releaseSettings, + exportJars := true, + Compile / packageBin := (flintSparkIntegration / assembly).value) + +lazy val releaseSettings = Seq( + publishMavenStyle := true, + publishArtifact := true, + Test / publishArtifact := false, + licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0")), + pomExtra := + https://opensearch.org/ + + git@github.com:opensearch-project/opensearch-spark.git + scm:git:git@github.com:opensearch-project/opensearch-spark.git + ) From ce8b2e46f25a9dbadae2655463ef3ef054de6cca Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Mon, 24 Jul 2023 14:19:40 -0700 Subject: [PATCH 06/11] update tracking branch Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 9f22632d2..022e0d415 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -5,7 +5,7 @@ on: push: branches: - main - - pulish-snapshot + - publish-snapshot jobs: build-and-publish-snapshots: From bfcfcd404bb18554f821b3f49826a787749a1f6f Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 25 Jul 2023 13:09:49 -0700 Subject: [PATCH 07/11] update m2 folder name Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 022e0d415..0a699532a 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -55,4 +55,5 @@ jobs: echo "::add-mask::$SONATYPE_USERNAME" echo "::add-mask::$SONATYPE_PASSWORD" export SNAPSHOT_REPO_URL="https://aws.oss.sonatype.org/content/repositories/snapshots/" - build/resources/publish/publish-snapshot.sh $HOME/.m2 + cd build/resources/publish/ + ./publish-snapshot.sh $HOME/.m2/repository/org/opensearch/opensearch-spark-standalone_2.12 From 88da1ce0ceffed2866ee878a3382f9c6e7036145 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 25 Jul 2023 13:30:29 -0700 Subject: [PATCH 08/11] update Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 0a699532a..7de9097c8 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -43,10 +43,10 @@ jobs: - name: generate sha and md5 run: | - for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" >> "$i.sha512"; done - for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha512sum "$i" >> "$i.sha512"; done - for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" >> "$i.sha256"; done - for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha256sum "$i" >> "$i.sha256"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha512sum "$i" | awk '{print $1}' >> "$i.sha512"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.pom" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done + for i in `find ${HOME}/.m2/repository/org/opensearch/ -name "*.jar" -type f`; do sha256sum "$i" | awk '{print $1}' >> "$i.sha256"; done - name: Get credentials and publish snapshots to maven run: | @@ -56,4 +56,5 @@ jobs: echo "::add-mask::$SONATYPE_PASSWORD" export SNAPSHOT_REPO_URL="https://aws.oss.sonatype.org/content/repositories/snapshots/" cd build/resources/publish/ - ./publish-snapshot.sh $HOME/.m2/repository/org/opensearch/opensearch-spark-standalone_2.12 + cp -a $HOME/.m2/repository/* ./ + ./publish-snapshot.sh ./ From d59236364cfe5d89eb4fae4ec68afb76c58d3ffb Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 25 Jul 2023 20:03:18 -0700 Subject: [PATCH 09/11] update to spark 3.3.2 Signed-off-by: Peng Huo --- build.sbt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index ac640cb2b..b6a232796 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ import Dependencies._ lazy val scala212 = "2.12.14" -lazy val sparkVersion = "3.3.1" +lazy val sparkVersion = "3.3.2" lazy val opensearchVersion = "2.6.0" ThisBuild / organization := "org.opensearch" @@ -33,6 +33,8 @@ lazy val compileScalastyle = taskKey[Unit]("compileScalastyle") lazy val testScalastyle = taskKey[Unit]("testScalastyle") lazy val commonSettings = Seq( + javacOptions ++= Seq("-source", "11"), + Compile / compile / javacOptions ++= Seq("-target", "11"), // Scalastyle scalastyleConfig := (ThisBuild / scalastyleConfig).value, compileScalastyle := (Compile / scalastyle).toTask("").value, @@ -48,6 +50,7 @@ lazy val root = (project in file(".")) lazy val flintCore = (project in file("flint-core")) .disablePlugins(AssemblyPlugin) .settings( + commonSettings, name := "flint-core", scalaVersion := scala212, libraryDependencies ++= Seq( From 04f92c614707ddc24a63b40bfc9625a170299840 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 28 Jul 2023 09:21:24 -0700 Subject: [PATCH 10/11] remove publish-snapshot branch from github actions Signed-off-by: Peng Huo --- .github/workflows/snapshot-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 7de9097c8..1e3367155 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - publish-snapshot jobs: build-and-publish-snapshots: From 94c376c3cfeb382470c4869bb72877dceb3ae9bf Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 28 Jul 2023 09:27:41 -0700 Subject: [PATCH 11/11] update readme Signed-off-by: Peng Huo --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c0a604b09..1b46e5e47 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,11 @@ spark-sql --conf "spark.sql.extensions=org.opensearch.flint.FlintSparkExtensions To build and run this application with Spark, you can run: ``` -sbt clean publishLocal +sbt clean standaloneCosmetic/publishM2 +``` +then add org.opensearch:opensearch-spark_2.12 when run spark application, for example, +``` +bin/spark-shell --packages "org.opensearch:opensearch-spark_2.12:0.1.0-SNAPSHOT" ``` ## Code of Conduct @@ -44,4 +48,4 @@ See the [LICENSE](../LICENSE.txt) file for our project's licensing. We will ask ## Copyright -Copyright OpenSearch Contributors. See [NOTICE](../NOTICE) for details. \ No newline at end of file +Copyright OpenSearch Contributors. See [NOTICE](../NOTICE) for details.