Load test only

.github/codecov.yml

@@ -16,16 +16,16 @@ component_management:
   - component_id: spanner-templates
     name: spanner-templates
     paths:
-    - "v1/src/main/java/com/google/cloud/teleport/spanner/**"
-    - "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
-    - "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
-    - "v2/datastream-to-spanner/**"
-    - "v2/spanner-common/**"
-    - "v2/spanner-change-streams-to-sharded-file-sink/**"
-    - "v2/gcs-to-sourcedb/**"
-    - "v2/spanner-migrations-sdk/**"
-    - "v2/spanner-custom-shard/**"
-    - "v2/sourcedb-to-spanner/**"
+    # - "v1/src/main/java/com/google/cloud/teleport/spanner/**"
+    # - "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
+    # - "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
+    # - "v2/datastream-to-spanner/**"
+    # - "v2/spanner-common/**"
+    # - "v2/spanner-change-streams-to-sharded-file-sink/**"
+    # - "v2/gcs-to-sourcedb/**"
+    # - "v2/spanner-migrations-sdk/**"
+    # - "v2/spanner-custom-shard/**"
+    # - "v2/sourcedb-to-spanner/**"
     - "v2/spanner-to-sourcedb/**"
     statuses:
     - type: project

.github/workflows/java-pr.yml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 # Checks that are intended to run on PRs containing Java code.
 
 name: Java PR
@@ -21,23 +20,12 @@ on:
     branches:
       - 'main'
     paths:
-      - '**.java'
-      - '**.xml'
-      # Include python files and Dockerfiles used for YAML and xlang templates.
-      - '**.py'
-      - 'plugins/core-plugin/src/main/resources/**'
-      # Include relevant GitHub Action files for running these checks.
-      # This will make it easier to verify action changes don't break anything.
+      - 'v2/spanner-to-sourcedb/**.java'
+      - 'v2/spanner-to-sourcedb/**.xml'
+      - 'v2/spanner-to-sourcedb/**.py'
       - '.github/actions/setup-env/*'
       - '.github/workflows/java-pr.yml'
-      # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml)
-      - '!v2/datastream-to-spanner/**'
-      - '!v2/spanner-common/**'
-      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
-      - '!v2/gcs-to-sourcedb/**'
-      - '!v2/spanner-migrations-sdk/**'
-      - '!v2/spanner-custom-shard/**'
-      - '!v2/sourcedb-to-spanner/**'
+      - '!v1/**'
   schedule:
     - cron: "0 */12 * * *"
   workflow_dispatch:
@@ -79,7 +67,7 @@ jobs:
   java_build:
     name: Build
     timeout-minutes: 60
-    runs-on: [self-hosted, it]
+    runs-on: [self-hosted, lt]
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -90,117 +78,120 @@ jobs:
         run: ./cicd/run-build
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
-  java_unit_tests:
-    name: Unit Tests
-    needs: [java_build]
-    timeout-minutes: 60
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Unit Tests
-        run: ./cicd/run-unit-tests
-      - name: Upload Unit Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Upload coverage reports to Codecov
-        uses: codecov/[email protected]
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: GoogleCloudPlatform/DataflowTemplates
-          files: 'target/site/jacoco-aggregate/jacoco.xml'
-          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
-          version: v0.6.0
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_smoke_tests_templates:
-    name: Dataflow Templates Integration Smoke Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
-    timeout-minutes: 60
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Integration Smoke Tests
-        run: | 
-          ./cicd/run-it-smoke-tests \
-          --modules-to-build="DEFAULT" \
-          --it-region="us-central1" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
-      - name: Upload Smoke Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_tests_templates:
-    name: Dataflow Templates Integration Tests
-    needs: [java_integration_smoke_tests_templates]
-    timeout-minutes: 240
+#  java_unit_tests:
+#    name: Unit Tests
+#    needs: [java_build]
+#    timeout-minutes: 60
+#    runs-on: [self-hosted, it]
+#    steps:
+#      - name: Checkout Code
+#        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+#      - name: Setup Environment
+#        id: setup-env
+#        uses: ./.github/actions/setup-env
+#      - name: Run Unit Tests
+#        run: ./cicd/run-unit-tests
+#        # run: mvn -B clean verify -f pom.xml -e -Dcheckstyle.skip -Djib.skip -DskipShade -Dspotless.check.skip -DskipIntegrationTests -fae -T8 --settings=.mvn/settings.xml -pl v2/spanner-to-sourcedb
+#      - name: Upload Unit Tests Report
+#        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+#        if: always() # always run even if the previous step fails
+#        with:
+#          name: surefire-test-results
+#          path: '**/surefire-reports/TEST-*.xml'
+#          retention-days: 1
+#      - name: Upload coverage reports to Codecov
+#        uses: codecov/[email protected]
+#        with:
+#          token: ${{ secrets.CODECOV_TOKEN }}
+#          slug: ollionorg/DataflowTemplates-fork
+#          files: 'target/site/jacoco-aggregate/jacoco.xml'
+#          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
+#          version: v0.6.0
+#      - name: Cleanup Java Environment
+#        uses: ./.github/actions/cleanup-java-env
+#        if: always()
+#  java_integration_smoke_tests_templates:
+#    name: Dataflow Templates Integration Smoke Tests
+#    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
+#    timeout-minutes: 60
+#    # Run on any runner that matches all the specified runs-on values.
+#    runs-on: [self-hosted, it]
+#    steps:
+#      - name: Checkout Code
+#        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+#      - name: Setup Environment
+#        id: setup-env
+#        uses: ./.github/actions/setup-env
+#      - name: Run Integration Smoke Tests
+#        run: |
+#          ./cicd/run-it-smoke-tests \
+#          --modules-to-build="v2/spanner-to-sourcedb" \
+#          --it-region="us-central1" \
+#          --it-project="cloud-teleport-testing" \
+#          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+#          --it-private-connectivity="datastream-private-connect-us-central1"
+#      - name: Upload Smoke Tests Report
+#        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+#        if: always() # always run even if the previous step fails
+#        with:
+#          name: surefire-test-results
+#          path: '**/surefire-reports/TEST-*.xml'
+#          retention-days: 1
+#      - name: Cleanup Java Environment
+#        uses: ./.github/actions/cleanup-java-env
+#        if: always()
+#  java_integration_tests_templates:
+#    name: Dataflow Templates Integration Tests
+#    needs: [spotless_check, checkstyle_check, java_build]
+#    timeout-minutes: 240
+#    # Run on any runner that matches all the specified runs-on values.
+#    runs-on: [self-hosted, it]
+#    steps:
+#      - name: Checkout Code
+#        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+#      - name: Setup Environment
+#        id: setup-env
+#        uses: ./.github/actions/setup-env
+#      - name: Run Integration Tests
+#        run: |
+#          ./cicd/run-it-tests \
+#          --modules-to-build="v2/spanner-to-sourcedb" \
+#          --it-region="asia-south1" \
+#          --it-project="daring-fiber-439305-v4" \
+#          --it-artifact-bucket="ollion-teleport-testing" \
+#          --it-private-connectivity="datastream-private-connect-asia-south1"
+#      - name: Upload Integration Tests Report
+#        uses: actions/upload-artifact@v4
+#        if: always() # always run even if the previous step fails
+#        with:
+#          name: surefire-test-results
+#          path: '**/surefire-reports/TEST-*.xml'
+#          retention-days: 1
+#      - name: Cleanup Java Environment
+#        uses: ./.github/actions/cleanup-java-env
+#        if: always()
+  java_load_tests_templates:
+    # if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
+    name: Dataflow Templates Load Tests
+    needs: [spotless_check, checkstyle_check, java_build]
+    # needs: [spotless_check, checkstyle_check]
+    timeout-minutes: 600
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
+    runs-on: [self-hosted, lt]
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
-      - name: Run Integration Tests
-        run: | 
-          ./cicd/run-it-tests \
-          --modules-to-build="DEFAULT" \
-          --it-region="us-central1" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
-      - name: Upload Integration Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
+      - name: Run Load Tests
+        run: |
+          ./cicd/run-load-tests \
+          --it-region="asia-south1" \
+          --it-project="daring-fiber-439305-v4" \
+          --it-artifact-bucket="ollion-teleport-testing" \
+          --it-private-connectivity="datastream-private-connect-asia-south1" \
+          --modules-to-build="v2/spanner-to-sourcedb"
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
         if: always()
-  java_load_tests_templates:
-    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
-    name: Dataflow Templates Load Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
-    timeout-minutes: 600
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, perf]
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-    - name: Setup Environment
-      id: setup-env
-      uses: ./.github/actions/setup-env
-    - name: Run Load Tests
-      run: |
-        ./cicd/run-load-tests \
-        --it-region="us-central1" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-private-connect-us-central1"
-    - name: Cleanup Java Environment
-      uses: ./.github/actions/cleanup-java-env
-      if: always()

.mvn/settings.xml

@@ -14,7 +14,7 @@
             </activation>
             <!-- Also update repo values in plugins/core-plugin/s.m.j.c.g.c.t/plugin/DockerfileGenerator.java-->
             <repositories>
-                <repository>
+<!--                 <repository>
                     <id>central</id>
                     <name>Maven Central remote repository</name>
                     <url>artifactregistry://us-maven.pkg.dev/artifact-foundry-prod/ah-3p-staging-maven</url>
@@ -25,7 +25,11 @@
                     <snapshots>
                         <enabled>true</enabled>
                     </snapshots>
-                </repository>
+                </repository> -->
+                <repository>
+                  <id>maven-central</id>
+                  <url>https://repo.maven.apache.org/maven2/</url>
+                </repository>      
             </repositories>
             <properties>
                 <airlockJavaRepo>ah-3p-staging-maven</airlockJavaRepo>
@@ -48,4 +52,4 @@
             <mirrorOf>external:artifactregistry:*</mirrorOf>
         </mirror>
     </mirrors>
-</settings>
+</settings>

cicd/cmd/run-it-smoke-tests/main.go

@@ -58,8 +58,8 @@ func main() {
 		mvnFlags.RunIntegrationSmokeTests(),
 		mvnFlags.ThreadCount(8),
 		mvnFlags.IntegrationTestParallelism(4),
-		mvnFlags.StaticBigtableInstance("teleport"),
-		mvnFlags.StaticSpannerInstance("teleport"),
+// 		mvnFlags.StaticBigtableInstance("teleport"),
+		mvnFlags.StaticSpannerInstance("spanner-demo"),
 		mvnFlags.InternalMaven(),
 		flags.Region(),
 		flags.Project(),
@@ -68,13 +68,13 @@ func main() {
 		flags.PrivateConnectivity(),
 		flags.SpannerHost(),
 		flags.FailureMode(),
-		flags.RetryFailures(),
-		flags.StaticOracleHost(),
-		flags.StaticOracleSysPassword(),
-		flags.CloudProxyHost(),
-		flags.CloudProxyMySqlPort(),
-		flags.CloudProxyPostgresPort(),
-		flags.CloudProxyPassword())
+		flags.RetryFailures())
+// 		flags.StaticOracleHost(),
+// 		flags.StaticOracleSysPassword(),
+// 		flags.CloudProxyHost(),
+// 		flags.CloudProxyMySqlPort(),
+// 		flags.CloudProxyPostgresPort(),
+// 		flags.CloudProxyPassword())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}