chore(ci): add license for integration test (#449)

.github/actions/setup/action.yml

@@ -1,6 +1,10 @@
 name: Setup Rust Builder
 description: 'Prepare Rust Build Environment'
 inputs:
+  runner:
+    description: 'runner provider'
+    required: true
+    default: 'github'
   cache-key:
     description: 'the rust cache key suffix'
     required: false
@@ -22,7 +26,8 @@ runs:
     - name: Setup sccache
       uses: mozilla-actions/[email protected]
 
-    - name: Cache Cargo
+    - name: Cache Cargo for Github Runner
+      if: inputs.runner == 'github'
       uses: actions/cache@v4
       with:
         path: |
@@ -33,6 +38,23 @@ runs:
           ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
           ${{ runner.os }}-cargo
 
+    - name: Cache Cargo for AWS Self-Hosted Runner
+      if: inputs.runner == 'aws'
+      uses: everpcpc/actions-cache@v2
+      env:
+        AWS_REGION: us-east-2
+      with:
+        provider: s3
+        bucket: "databend-ci"
+        root: "bendsql/cache"
+        path: |
+          ~/.cargo/registry/
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ steps.toolchain.outputs.cachekey }}-${{ inputs.cache-key }}
+        restore-keys: |
+          ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ steps.toolchain.outputs.cachekey }}
+          ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+          ${{ runner.os }}-cargo
+
     - name: Setup rust related environment variables
       shell: bash
       run: |
@@ -48,6 +70,20 @@ runs:
         echo "RUST_LOG=info" >> $GITHUB_ENV
         # Enable sparse index
         echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> $GITHUB_ENV
-        # Enable sccache
+
+    - name: Enable sccache for Github Actions
+      shell: bash
+      if: inputs.runner == 'github'
+      run: |
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+
+    - name: Enable sccache for AWS Self-Hosted Actions
+      shell: bash
+      if: inputs.runner == 'aws'
+      run: |
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        echo "SCCACHE_BUCKET=databend-ci" >> $GITHUB_ENV
+        echo "SCCACHE_REGION=us-east-2" >> $GITHUB_ENV
+        echo "SCCACHE_S3_KEY_PREFIX=bendsql/sccache/" >> $GITHUB_ENV
+        echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV

.github/workflows/ci.yml

@@ -50,12 +50,19 @@ jobs:
 
   integration:
     needs: check
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, X64, 4c8g, aws]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
         with:
+          runner: aws
           cache-key: integration
+      - name: Get License from S3
+        run: |
+          aws s3 cp s3://databend-ci/misc/license.key license.key
+          aws s3 cp s3://databend-ci/misc/license.json license.json
+          cat license.json
+          echo "QUERY_DATABEND_ENTERPRISE_LICENSE=$(cat license.key)" >> $GITHUB_ENV
       - run: make -C tests test-core
       - run: make -C tests test-driver
       - run: make -C tests test-bendsql

cli/test.sh

@@ -29,23 +29,9 @@ case $TEST_HANDLER in
 "flight")
 	echo "==> Testing Flight SQL handler"
 	export BENDSQL_DSN="databend+flight://${DATABEND_USER}:${DATABEND_PASSWORD}@${DATABEND_HOST}:8900/?sslmode=disable"
-	export BENDSQL="${CARGO_TARGET_DIR}/debug/bendsql"
 	;;
 "http")
 	echo "==> Testing REST API handler"
-	export BENDSQL="${CARGO_TARGET_DIR}/debug/bendsql"
-
-	echo "create user if not exists databend identified by 'databend'" |  $BENDSQL -dsn="databend+http://${DATABEND_USER}:${DATABEND_PASSWORD}@${DATABEND_HOST}:8000/?sslmode=disable&presign=on"
-	echo "grant all on *.* to databend" |  $BENDSQL -dsn="databend+http://${DATABEND_USER}:${DATABEND_PASSWORD}@${DATABEND_HOST}:8000/?sslmode=disable&presign=on"
-
-	export BENDSQL_NEW="${BENDSQL} --user databend --password databend --host ${DATABEND_HOST} --port 8000"
-
-	$BENDSQL_NEW --query="select 1 from numbers(10) where number > 1000"
-	$BENDSQL_NEW --query="create database if not exists aaa"
-	$BENDSQL_NEW -D aaa --query="create table if not exists bbb(a int)"
-	$BENDSQL_NEW -D aaa --query="drop table bbb"
-	$BENDSQL_NEW -D default --query="drop database aaa"
-
 	export BENDSQL_DSN="databend+http://${DATABEND_USER}:${DATABEND_PASSWORD}@${DATABEND_HOST}:8000/?sslmode=disable&presign=on"
 	;;
 *)
@@ -54,6 +40,8 @@ case $TEST_HANDLER in
 	;;
 esac
 
+export BENDSQL="${CARGO_TARGET_DIR}/debug/bendsql"
+
 for tf in cli/tests/*.{sql,sh}; do
 	[[ -e "$tf" ]] || continue
 	echo "    Running test -- ${tf}"

cli/tests/http/05-stream.sh

@@ -1,22 +1,21 @@
 #!/bin/bash
-## uncomment these when QUERY_DATABEND_ENTERPRISE_LICENSE env is set 
 
-# cat <<SQL | ${BENDSQL}
-# CREATE DATABASE IF NOT EXISTS stream_test;
+cat <<SQL | ${BENDSQL}
+CREATE DATABASE IF NOT EXISTS stream_test;
 
-# CREATE OR REPLACE TABLE stream_test.abc
-# (
-#     title VARCHAR,
-#     author VARCHAR,
-#     date VARCHAR
-# );
+CREATE OR REPLACE TABLE stream_test.abc
+(
+    title VARCHAR,
+    author VARCHAR,
+    date VARCHAR
+);
 
-# CREATE OR REPLACE STREAM stream_test.s on table stream_test.abc;
-# SQL
+CREATE OR REPLACE STREAM stream_test.s on table stream_test.abc;
+SQL
 
-# cat <<SQL | ${BENDSQL} -D stream_test
-# DROP TABLE abc
-# SQL
+cat <<SQL | ${BENDSQL} -D stream_test
+DROP TABLE abc
+SQL
 
 cat <<SQL | ${BENDSQL} -D stream_test
 select 1;

tests/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 services:
   minio:
     image: docker.io/minio/minio
@@ -7,13 +7,10 @@ services:
     volumes:
       - ./data:/data
   databend:
-    # Use nightly image for testing
-    #
-    # Please remove the nightly tag once https://github.com/datafuselabs/databend/pull/15559 included
-    # in stable.
-    image: docker.io/datafuselabs/databend:nightly
+    image: docker.io/datafuselabs/databend
     environment:
       - QUERY_STORAGE_TYPE=s3
+      - QUERY_DATABEND_ENTERPRISE_LICENSE
       - AWS_S3_ENDPOINT=http://localhost:9000
       - AWS_ACCESS_KEY_ID=minioadmin
       - AWS_SECRET_ACCESS_KEY=minioadmin