Fix nightly CI for NativeTest under GraalVM CE

.github/workflows/graalvm.yml

@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: NativeTest CI - GraalVM Native Image
+
+# Only execute CI when changes involving GraalVM Reachability Metadata and nativeTest are involved. Because most Contributors don't use GraalVM CE.
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+      - '.github/workflows/graalvm.yml'
+      - 'infra/reachability-metadata/src/**'
+      - 'test/native/native-image-filter/**'
+      - 'test/native/src/**'
+
+concurrency:
+  group: graalvm-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
+
+jobs:
+  nativetest:
+    if: github.repository == 'apache/shardingsphere'
+    name: GraalVM - GraalVM CE for JDK ${{ matrix.java-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        java-version: [ '17.0.9' ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'graalvm-community'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          native-image-pr-reports: true
+      - name: Run nativeTest with GraalVM CE for ${{ matrix.java-version }}
+        run: ./mvnw -PnativeTestInShardingSphere -T1C -e clean test

.github/workflows/nightly-ci.yml

@@ -102,7 +102,7 @@ jobs:
 
   ci-native-test:
     if: github.repository == 'apache/shardingsphere'
-    name: NativeTest CI - GraalVM CE on ${{ matrix.os }}
+    name: CI - NativeTest on GraalVM CE for JDK ${{ matrix.java-version }} on ${{ matrix.os }}
     needs: global-environment
     runs-on: ${{ matrix.os }}
     timeout-minutes: 90
@@ -111,11 +111,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest ]
+        java-version: [ '17.0.9' ]
     steps:
       - uses: actions/checkout@v3
       - uses: graalvm/setup-graalvm@v1
         with:
-          java-version: '17.0.9'
+          java-version: ${{ matrix.java-version }}
           distribution: 'graalvm-community'
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/cache@v3

infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/proxy-config.json

@@ -0,0 +1,6 @@
+[
+  {
+    "condition":{"typeReachable":"org.apache.shardingsphere.infra.state.datasource.DataSourceStateManager"},
+    "interfaces":["java.sql.Connection"]
+  }
+]

infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/resource-config.json

@@ -1,146 +1,17 @@
 {
   "resources":{
   "includes":[{
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.schema.builder.SystemSchemaBuilderRule"},
+    "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.schema.builder.SystemSchemaBuilder"},
     "pattern":".*schema/.+\\.yaml$"
   }, {
     "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"},
     "pattern":".*sql/.+\\.xml$"
+  }, {
+    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader"},
+    "pattern":".*META-INF/services/org\\.apache\\.shardingsphere\\..+"
   }, {
     "condition":{"typeReachable":"org.apache.shardingsphere.infra.state.datasource.DataSourceStateManager"},
     "pattern":"\\QMETA-INF/services/java.sql.Driver\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.NewClusterContextManagerBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.authority.spi.AuthorityRegistryProvider\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.driver.state.DriverState\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.config.rule.checker.RuleConfigurationChecker\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.metadata.persist.NewMetaDataPersistService"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.config.rule.decorator.RuleConfigurationDecorator\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.connection.refresher.MetaDataRefresher\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.connector.ConnectionPropertiesParser\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.metadata.data.loader.MetaDataLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.metadata.data.loader.DialectMetaDataLoader\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeRegistry"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.metadata.database.DialectDatabaseMetaData\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.metadata.database.system.SystemDatabase"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.metadata.database.system.DialectSystemDatabase\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.database.core.type.DatabaseTypeFactory"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.type.DatabaseType\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.datanode.DataNodes"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.datanode.DataNodeBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.datasource.pool.props.domain.DataSourcePoolProperties"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.datasource.pool.metadata.DataSourcePoolMetaData\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.audit.SQLAuditEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.audit.SQLAuditor\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.sql.prepare.driver.DriverExecutionPrepareEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.executor.sql.prepare.driver.SQLExecutionUnitBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.core.InlineExpressionParserFactory"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.expr.spi.InlineExpressionParser\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.instance.metadata.InstanceMetaDataBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.merge.MergeEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.merge.engine.ResultProcessEngine\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.metadata.database.schema.builder.GenericSchemaBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.metadata.database.schema.reviser.MetaDataReviseEntry\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.NewClusterContextManagerBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.metadata.statistics.builder.ShardingSphereStatisticsBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.rewrite.SQLRewriteEntry"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.rewrite.context.SQLRewriteContextDecorator\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.route.engine.impl.PartialSQLRouteExecutor"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.route.SQLRouter\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.rule.builder.database.DatabaseRulesBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.rule.builder.database.DatabaseRuleBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.rule.builder.database.DatabaseRulesBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.rule.builder.database.DefaultDatabaseRuleConfigurationBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.rule.builder.global.GlobalRulesBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.rule.builder.global.DefaultGlobalRuleConfigurationBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.rule.builder.global.GlobalRulesBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.rule.builder.global.GlobalRuleBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.util.yaml.constructor.ShardingSphereYamlConstruct\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.util.yaml.shortcuts.ShardingSphereYamlShortcuts\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.swapper.mode.YamlModeConfigurationSwapper"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.yaml.config.swapper.mode.YamlPersistRepositoryConfigurationSwapper\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.swapper.rule.NewYamlGlobalRuleConfigurationSwapperEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.yaml.config.swapper.rule.NewYamlGlobalRuleConfigurationSwapper\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.yaml.config.swapper.rule.NewYamlRuleConfigurationSwapperEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.yaml.config.swapper.rule.NewYamlRuleConfigurationSwapper\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.manager.ContextManagerBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.coordinator.registry.NewGovernanceWatcherFactory"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.manager.cluster.coordinator.registry.NewGovernanceWatcher\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.NewClusterContextManagerBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.repository.cluster.ClusterPersistRepository\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.cluster.lock.holder.DistributedLockHolder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.repository.cluster.lock.creator.DistributedLockCreator\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.repository.standalone.StandalonePersistRepository\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.metadata.builder.RuleConfigurationEventBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.mode.spi.RuleNodePathProvider\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.readwritesplitting.route.standard.StandardReadwriteSplittingDataSourceRouter"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.readwritesplitting.route.standard.filter.ReadDataSourcesFilter\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.readwritesplitting.spi.ReadQueryLoadBalanceAlgorithm\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.sql.parser.spi.SQLDialectParserFacade\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.sql.parser.spi.SQLStatementVisitorFacade\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.optimizer.context.parser.dialect.OptimizerSQLPropertiesBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.sqlfederation.optimizer.context.parser.dialect.OptimizerSQLDialectBuilder\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.sqlfederation.engine.SQLFederationEngine"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.sqlfederation.spi.SQLFederationDecider\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.sqltranslator.rule.SQLTranslatorRule"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.sqltranslator.spi.SQLTranslator\\E"
-  }, {
-    "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.NewClusterContextManagerBuilder"},
-    "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.timeservice.spi.TimestampService\\E"
   }, {
     "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.sql.JDBCRepositorySQLLoader"},
     "pattern":"\\Qsql\\E"

test/native/pom.xml

@@ -81,6 +81,11 @@
             <artifactId>mssqlserver</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-test</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/mode/cluster/ZookeeperTest.java

@@ -17,47 +17,38 @@
 
 package org.apache.shardingsphere.test.natived.jdbc.mode.cluster;
 
-import com.github.dockerjava.api.model.ExposedPort;
-import com.github.dockerjava.api.model.HostConfig;
-import com.github.dockerjava.api.model.PortBinding;
-import com.github.dockerjava.api.model.Ports;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingServer;
 import org.apache.shardingsphere.driver.api.yaml.YamlShardingSphereDataSourceFactory;
 import org.apache.shardingsphere.test.natived.jdbc.commons.FileTestUtils;
 import org.apache.shardingsphere.test.natived.jdbc.commons.TestShardingService;
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledInNativeImage;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.utility.DockerImageName;
 
 import javax.sql.DataSource;
-import java.io.IOException;
 import java.sql.SQLException;
 import java.time.Duration;
 import java.util.concurrent.TimeUnit;
 
 class ZookeeperTest {
 
+    private static final int ZOOKEEPER_SERVER_HOST_PORT = 62372;
+
+    private static TestingServer testingServer;
+
     private TestShardingService testShardingService;
 
-    @SuppressWarnings("resource")
     @Test
-    @EnabledInNativeImage
-    void assertShardingInLocalTransactions() throws SQLException, IOException {
-        try (
-                GenericContainer<?> zookeeperContainer = new GenericContainer<>(DockerImageName.parse("zookeeper:3.9.1-jre-17"))
-                        .withCreateContainerCmdModifier(cmd -> cmd.withHostConfig(new HostConfig().withPortBindings(new PortBinding(Ports.Binding.bindPort(62372), new ExposedPort(2181)))))) {
-            zookeeperContainer.start();
-            this.beforeAll();
-            DataSource dataSource = YamlShardingSphereDataSourceFactory.createDataSource(FileTestUtils.readFromFileURLString("test-native/yaml/mode/cluster/zookeeper.yaml"));
-            testShardingService = new TestShardingService(dataSource);
-            this.initEnvironment();
-            testShardingService.processSuccess();
-            testShardingService.cleanEnvironment();
-        }
+    void assertShardingInLocalTransactions() throws Exception {
+        beforeAll();
+        DataSource dataSource = YamlShardingSphereDataSourceFactory.createDataSource(FileTestUtils.readFromFileURLString("test-native/yaml/mode/cluster/zookeeper.yaml"));
+        testShardingService = new TestShardingService(dataSource);
+        initEnvironment();
+        testShardingService.processSuccess();
+        testShardingService.cleanEnvironment();
+        afterAll();
     }
 
     private void initEnvironment() throws SQLException {
@@ -69,13 +60,18 @@ private void initEnvironment() throws SQLException {
         testShardingService.getAddressRepository().truncateTable();
     }
 
-    private void beforeAll() {
-        Awaitility.await().atMost(Duration.ofMinutes(1)).ignoreExceptions().until(() -> {
-            CuratorFramework client = CuratorFrameworkFactory.newClient("localhost:" + 62372, new ExponentialBackoffRetry(1000, 3));
+    private void beforeAll() throws Exception {
+        testingServer = new TestingServer(ZOOKEEPER_SERVER_HOST_PORT, true);
+        Awaitility.await().atMost(Duration.ofSeconds(30L)).ignoreExceptions().until(() -> {
+            CuratorFramework client = CuratorFrameworkFactory.newClient("localhost:" + ZOOKEEPER_SERVER_HOST_PORT, new ExponentialBackoffRetry(1000, 3));
             client.start();
             boolean connected = client.blockUntilConnected(5, TimeUnit.SECONDS);
             client.close();
             return connected;
         });
     }
+
+    private void afterAll() throws Exception {
+        testingServer.close();
+    }
 }