Enhancement: Add e2e test module. (#5)

.github/workflows/push_pr.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - ".*"
+  push:
+    branches:
+      - main
+      - 'release-*'
+
+concurrency:
+  group: ci-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  spark-connector-oceanbase:
+    uses: ./.github/workflows/test.yml
+    with:
+      module: spark-connector-oceanbase/spark-connector-oceanbase-base
+
+  e2e-tests:
+    strategy:
+      matrix:
+        spark_version: ["3.1","3.2","3.3","3.4"]
+    uses: ./.github/workflows/test.yml
+    with:
+      module: spark-connector-oceanbase-e2e-tests
+      maven_opts: "-D spark_version=${{ matrix.spark_version }}"
+
+  e2e-tests-scala-2_11:
+    strategy:
+      matrix:
+        spark_version: ["2.4.6"]
+    uses: ./.github/workflows/test_scala2_11.yml
+    with:
+      module: spark-connector-oceanbase-e2e-tests
+      maven_opts: "-D spark_version=${{ matrix.spark_version }}"

.github/workflows/test.yml

@@ -0,0 +1,42 @@
+name: Test Module
+
+on:
+  workflow_call:
+    inputs:
+      module:
+        required: true
+        type: string
+      maven_opts:
+        required: false
+        type: string
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'zulu'
+          cache: 'maven'
+
+      - name: Maven build
+        run: mvn install -DskipTests=true
+
+      - name: Maven test
+        run: mvn verify -pl ${{ inputs.module }} -am ${{ inputs.maven_opts }}

.github/workflows/test_scala2_11.yml

@@ -0,0 +1,42 @@
+name: Test Module
+
+on:
+  workflow_call:
+    inputs:
+      module:
+        required: true
+        type: string
+      maven_opts:
+        required: false
+        type: string
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'zulu'
+          cache: 'maven'
+
+      - name: Maven build
+        run: mvn clean install -D scala.version=2.11.12 -D scala.binary.version=2.11 -D skipTests=true
+
+      - name: Maven test
+        run: mvn verify -pl ${{ inputs.module }} -am ${{ inputs.maven_opts }} -D scala.binary.version=2.11

pom.xml

@@ -24,6 +24,7 @@ under the License.
     <modules>
         <module>spark-connector-oceanbase</module>
         <module>spark-connector-oceanbase-common</module>
+        <module>spark-connector-oceanbase-e2e-tests</module>
     </modules>
 
     <properties>
@@ -67,6 +68,24 @@ under the License.
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+
+            <dependency>
+                <groupId>mysql</groupId>
+                <artifactId>mysql-connector-java</artifactId>
+                <version>8.0.28</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.protobuf</groupId>
+                        <artifactId>protobuf-java</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+            <dependency>
+                <groupId>com.oceanbase</groupId>
+                <artifactId>oceanbase-client</artifactId>
+                <version>2.4.12</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

spark-connector-oceanbase-common/src/test/java/com/oceanbase/spark/OceanBaseMySQLTestBase.java

@@ -47,7 +47,7 @@ public abstract class OceanBaseMySQLTestBase extends OceanBaseTestBase {
     private static final String SYS_PASSWORD = "123456";
     private static final String TEST_PASSWORD = "654321";
 
-    private static final Network NETWORK = Network.newNetwork();
+    public static final Network NETWORK = Network.newNetwork();
 
     @SuppressWarnings("resource")
     public static final GenericContainer<?> CONFIG_SERVER =
@@ -133,6 +133,24 @@ public static void createSysUser(String user, String password) throws SQLExcepti
         }
     }
 
+    private static String configServerAddress;
+    private static String obServerIP;
+
+    public static String getConfigServerAddress() {
+        if (configServerAddress == null) {
+            String ip = getContainerIP(CONFIG_SERVER);
+            configServerAddress = "http://" + ip + ":" + CONFIG_SERVER_PORT;
+        }
+        return configServerAddress;
+    }
+
+    public static String getOBServerIP() {
+        if (obServerIP == null) {
+            obServerIP = getContainerIP(CONTAINER);
+        }
+        return obServerIP;
+    }
+
     @Override
     public String getHost() {
         return CONTAINER.getHost();