no-snow Parallelize e2e jar tests (#767)

snowflakedb · Jun 11, 2024 · bbf9291 · bbf9291
1 parent aecbc30
commit bbf9291
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 76 deletions.
diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
@@ -68,12 +68,14 @@ jobs:
         continue-on-error: false
         run: mvn -DghActionsIT verify --batch-mode
   build-e2e-jar-test:
-    name: E2E JAR Test - Cloud ${{ matrix.snowflake_cloud }}
+    name: e2e-jar-test cloud=${{ matrix.snowflake_cloud }} test_type=${{ matrix.test_type }} java=${{ matrix.java_path_env_var }}
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ]
+        test_type: [ 'shaded', 'unshaded', 'fips']
+        java_path_env_var: [ "JAVA_HOME_8_X64", "JAVA_HOME_11_X64", "JAVA_HOME_17_X64", "JAVA_HOME_21_X64" ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -92,4 +94,7 @@ jobs:
           DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
         run: ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
       - name: Run E2E JAR Test
+        env:
+          test_type: ${{ matrix.test_type }}
+          java_path_env_var: ${{ matrix.java_path_env_var }}
         run: ./e2e-jar-test/run_e2e_jar_test.sh
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
@@ -29,7 +29,7 @@
       <dependency>
         <groupId>net.snowflake</groupId>
         <artifactId>snowflake-ingest-sdk</artifactId>
-        <version>2.1.1</version>
+        <version>2.1.2-SNAPSHOT</version>
       </dependency>
 
       <dependency>

diff --git a/e2e-jar-test/run_e2e_jar_test.sh b/e2e-jar-test/run_e2e_jar_test.sh
@@ -2,101 +2,81 @@
 
 set -euo pipefail
 
+maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk"
+
 ## This script tests the SDK JARs end-to-end, i.e. not using integration tests from within the project, but from an
 ## external Maven project, which depends on the SDK deployed into the local maven repository. The following SDK variants are tested:
 ## 1. Shaded jar
 ## 2. Unshaded jar
 ## 3. FIPS-compliant jar, i.e. unshaded jar without snowflake-jdbc and bouncy castle dependencies, but with snowflake-jdbc-fips depedency
 
-maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
-sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk"
-
 cp profile.json e2e-jar-test/standard
 cp profile.json e2e-jar-test/fips
 
-echo "###################"
-echo "# TEST SHADED JAR #"
-echo "###################"
-
-# Remove the SDK from local maven repository
-rm -fr $sdk_repo_dir
-
-# Prepare pom.xml for shaded JAR
-project_version=$(./scripts/get_project_info_from_pom.py pom.xml version)
-./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
-
-# Build shaded SDK always with Java 8
-echo "Building shaded SDK"
-mvn clean package -DskipTests=true --batch-mode --show-version
-
-# Install shaded SDK JARs into local maven repository
-mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml
-
-# Run e2e tests with all supported LTS Java versions
-cd e2e-jar-test
-
-echo "Testing shaded JAR with Java 8"
-mvn --show-version clean verify -pl standard -am # Test with Java 8
-
-echo "Testing shaded JAR with Java 11"
-JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11
+test_type="${test_type:-}"
+java_path_for_test_execution="${!java_path_env_var}"
 
-echo "Testing shaded JAR with Java 17"
-JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17
+# Cleanup ingest SDK in the local maven repository
+rm -fr "${sdk_repo_dir}"
 
-echo "Testing shaded JAR with Java 21"
-JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
-cd ..
+if [[ "${test_type}" == "shaded" ]]; then
+  echo "###################"
+  echo "# TEST SHADED JAR #"
+  echo "###################"
 
-echo "#####################"
-echo "# TEST UNSHADED JAR #"
-echo "#####################"
+  # Prepare pom.xml for shaded JAR
+  project_version=$(./scripts/get_project_info_from_pom.py pom.xml version)
+  ./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
 
-# Remove the SDK from local maven repository
-rm -r $sdk_repo_dir
+  # Build shaded SDK always with Java 8
+  echo "Building shaded SDK"
+  JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean package -DskipTests=true --batch-mode --show-version
 
-# Install unshaded SDK into local maven repository with Java 8
-echo "Building unshaded SDK"
-mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
+  # Install shaded SDK JARs into local maven repository
+  JAVA_HOME="${JAVA_HOME_8_X64}" mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml
 
-# Run e2e tests with all supported LTS Java versions
-cd e2e-jar-test
+  # Run e2e tests with all supported LTS Java versions
+  cd e2e-jar-test
 
-echo "Testing unshaded JAR with Java 8"
-mvn --show-version clean verify -pl standard -am # Test with Java 8
+  echo "Testing shaded JAR with ${java_path_for_test_execution}"
+  JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
+  cd ..
+elif [[ "${test_type}" == "unshaded" ]]; then
+  echo "#####################"
+  echo "# TEST UNSHADED JAR #"
+  echo "#####################"
 
-echo "Testing unshaded JAR with Java 11"
-JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11
+  # Install unshaded SDK into local maven repository with Java 8
+  echo "Building unshaded SDK"
+  JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
 
-echo "Testing unshaded JAR with Java 17"
-JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17
+  # Run e2e tests with all supported LTS Java versions
+  cd e2e-jar-test
 
-echo "Testing unshaded JAR with Java 21"
-JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
-cd ..
+  echo "Testing unshaded JAR with ${java_path_for_test_execution}"
+  JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
+  cd ..
+elif [[ "${test_type}" == "fips" ]]; then
+  echo "##############"
+  echo "# TEST FIPS #"
+  echo "##############"
+  # Install unshaded SDK into local maven repository with Java
+  echo "Building unshaded SDK for FIPS"
+  JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
 
-echo "##############"
-echo "# TEST FIPS #"
-echo "##############"
-# Remove the SDK from local maven repository
-rm -r $sdk_repo_dir
+  # Run e2e tests on the FIPS module with all supported LTS Java versions
+  cd e2e-jar-test
 
-# Install unshaded SDK into local maven repository with Java
-echo "Building unshaded SDK for FIPS"
-mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
+  echo "Testing FIPS JAR with ${java_path_for_test_execution}"
+  JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
+  cd ..
+else
+  echo "Unsupported value of \$test_type variable: \"${test_type}\"" >&2
+  exit 1
+fi
 
-# Run e2e tests on the FIPS module with all supported LTS Java versions
-cd e2e-jar-test
 
-echo "Testing FIPS JAR with Java 8"
-mvn --show-version clean verify -pl standard -am # Test with Java 8
 
-echo "Testing FIPS JAR with Java 11"
-JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11
 
-echo "Testing FIPS JAR with Java 17"
-JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17
 
-echo "Testing FIPS JAR with Java 21"
-JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
-cd ..
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.1</version>
+  <version>2.1.2-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.1";
+  public static final String DEFAULT_VERSION = "2.1.2-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";