diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 119698e97..39f5b8790 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -32,11 +32,11 @@ body:
attributes:
label: Search before asking
description: >
- Please make sure to search in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22)
+ Please make sure to search in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22)
first to see whether the same issue was reported already.
options:
- label: >
- I had searched in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found
+ I had searched in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found
no similar issues.
required: true
diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml
index 2a9877e3f..3d4350c4a 100644
--- a/.github/ISSUE_TEMPLATE/feature-request.yml
+++ b/.github/ISSUE_TEMPLATE/feature-request.yml
@@ -30,11 +30,11 @@ body:
attributes:
label: Search before asking
description: >
- Please make sure to search in the [feature](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) first
+ Please make sure to search in the [feature](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) first
to see whether the same feature was requested already.
options:
- label: >
- I had searched in the [feature](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) and found no
+ I had searched in the [feature](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) and found no
similar feature requirement.
required: true
diff --git a/.github/ISSUE_TEMPLATE/umbrella.yml b/.github/ISSUE_TEMPLATE/umbrella.yml
index 8db9b4dc1..af553ece2 100644
--- a/.github/ISSUE_TEMPLATE/umbrella.yml
+++ b/.github/ISSUE_TEMPLATE/umbrella.yml
@@ -34,11 +34,11 @@ body:
attributes:
label: Search before asking
description: >
- Please make sure to search in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22)
+ Please make sure to search in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22)
first to see whether the same issue was reported already.
options:
- label: >
- I had searched in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found
+ I had searched in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found
no similar issues.
required: true
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 3752eb8b0..66f08d31c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,7 +8,7 @@ Feel free to ping committers for the review!
## Contribution Checklist
- - Make sure that the pull request corresponds to a [GITHUB issue](https://github.com/apache/incubator-seatunnel/issues).
+ - Make sure that the pull request corresponds to a [GITHUB issue](https://github.com/apache/seatunnel/issues).
- Name the pull request in the form "[Feature] [component] Title of the pull request", where *Feature* can be replaced by `Hotfix`, `Bug`, etc.
@@ -24,5 +24,5 @@ Feel free to ping committers for the review!
* [ ] Code changed are covered with tests, or it does not need tests for reason:
* [ ] If any new Jar binary package adding in your PR, please add License Notice according
- [New License Guide](https://github.com/apache/incubator-seatunnel/blob/dev/docs/en/contribution/new-license.md)
-* [ ] If necessary, please update the documentation to describe the new feature. https://github.com/apache/incubator-seatunnel/tree/dev/docs
+ [New License Guide](https://github.com/apache/seatunnel/blob/dev/docs/en/contribution/new-license.md)
+* [ ] If necessary, please update the documentation to describe the new feature. https://github.com/apache/seatunnel/tree/dev/docs
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index d2ff663a1..05070c59b 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -131,7 +131,7 @@ jobs:
cache: 'maven'
- name: Install
run: >-
- ./mvnw -B -q install -DskipTests -P release
+ ./mvnw -B -q install -DskipTests -Prelease
-D"maven.test.skip"=true
-D"maven.javadoc.skip"=true
-D"checkstyle.skip"=true
diff --git a/README.md b/README.md
index 73ab7e35d..0eae9fc6b 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
-[![Backend Workflow](https://github.com/apache/incubator-seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/incubator-seatunnel/actions/workflows/backend.yml)
+[![Backend Workflow](https://github.com/apache/seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/seatunnel/actions/workflows/backend.yml)
[![Slack](https://img.shields.io/badge/slack-%23seatunnel-4f8eba?logo=slack)](https://join.slack.com/t/apacheseatunnel/shared_invite/zt-123jmewxe-RjB_DW3M3gV~xL91pZ0oVQ)
[![Twitter Follow](https://img.shields.io/twitter/follow/ASFSeaTunnel.svg?label=Follow&logo=twitter)](https://twitter.com/ASFSeaTunnel)
@@ -16,7 +16,7 @@ SeaTunnel was formerly named Waterdrop , and renamed SeaTunnel since October 12,
So, What we are?
An open-source web console to manage your seatunnel-script, and would push them to any scheduling-system easily.
-Click it if your want to know more about our design. 👉🏻[Design](https://github.com/apache/incubator-seatunnel/issues/1947)
+Click it if your want to know more about our design. 👉🏻[Design](https://github.com/apache/seatunnel/issues/1947)
## How to start
@@ -25,28 +25,37 @@ Notice: Some details please refer to the docs/QuickStart.md
### 1 Preparing the Apache SeaTunnel environment
-#### 1.1 Install Apache SeaTunnel
+Because SeaTunnel Web uses the SeaTunnel Java Client to submit jobs, running SeaTunnel Web requires preparing a SeaTunnel Zeta Engine service first.
-If you already have Apache SeaTunnel environment, you can skip this step and go to [Install your SeaTunnel to your local maven repository](#1.2 Install your SeaTunnel to your local maven repository)
+Based on the usage requirements of SeaTunnel Zeta Engine, the SeaTunnel Client node that submits the job must have the same operating system and installation directory structure as the SeaTunnel Server node that runs the job. Therefore, if you want to run SeaTunnel Web in IDEA, you must install and run a SeaTunnel Zeta Engine Server on the same machine as the IDEA.
-Because running SeaTunnel Web must rely on the SeaTunnel, if you do not have a SeaTunnel environment, you need to first install and deploy a SeaTunnel (hereinafter referred to as ST). You need to use the dev branch in the ST repositories.
+Don't worry, the next steps will tell you how to correctly install SeaTunnel Zeta Engine Server in different situations.
-Reference `https://github.com/apache/seatunnel` to deploy a ST
-
-#### 1.2 Install your SeaTunnel to your local maven repository
+### 2 Run SeaTunnel Web in IDEA
-If you already have a ST environment which is from ST github dev branch. Make sure the version is 2.3.2-SNAPSHOT in the pom file. then use command './mvnw clean install '-Dmaven.test.skip=true' '-Dcheckstyle.skip=true'' to install ST package into your local maven repository
+If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server)
+#### 2.1 Install SeaTunnel Zeta Engine Server
+You have two ways to get the SeaTunnel installer package. Build from source code or download from the SeaTunnel website.
-#### 1.3 Run SeaTunnel in RDEA
+**The SeaTunnel version used here is only for writing this document to show you the process used, and does not necessarily represent the correct version. SeaTunnel Web and SeaTunnel Engine have strict version dependencies, and you can confirm the specific version mapping through xxx. Now only support build SeaTunnel Web and Seatunnel Zeta Engine in local, Because it is necessary to ensure that the seatunnel-api in SeaTunnel Web and the version in SeaTunnel Zeta Engine are the same.**
-Reference `https://github.com/apache/seatunnel`
+##### 2.1.1 Build from source code and deploy
+* Get the source package from https://seatunnel.apache.org/download or https://github.com/apache/seatunnel.git
+* Build installer package use maven command `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" `
+* After building, it is necessary to set an environment variable `ST_WEB_BASEDIR_PATH` to represent the location of the data source shade package. A custom class loader will be used to load the data source shade package based on this. For example: `ST_WEB_BASEDIR_PATH=/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/`
+* Then you can get the installer package in `${Your_code_dir}/seatunnel-dist/target`, For example:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`
+* Run `tar -zxvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` to unzip the installer package.
+* Run `cd apache-seatunnel-2.3.3-SNAPSHOT & sh bin/seatunnel-cluster.sh -d` to run the SeaTunnel Zeta Engine Server.
+* Please confirm that port 5801 is being monitored by the SeaTunnelServer process.
-### 2 Run SeaTunnel Web in IDEA
+##### 2.1.2 Download installer package and deploy
+The other way to install SeaTunnel Zeta Engine Server is download the installer package from https://seatunnel.apache.org/download and deploy.
-If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server)
+* Download and install connector plugin(Some third-party dependency packages will also be automatically downloaded and installed during this process, such as hadoop jar). You can get the step from https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment.
+* Run `cd apache-seatunnel-2.3.2 & sh bin/seatunnel-cluster.sh -d` to run the SeaTunnel Zeta Engine Server.
-#### 2.1 Init database
+#### 2.2 Init database
1. Edit `seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_env.sh` file, Complete the installed database address, port, username, and password. Here is an example:
@@ -58,14 +67,27 @@ If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web
```
2. Run init shell `sh seatunnel-server/seatunnel-app/src/main/resources/script/init_sql.sh` If there are no errors during operation, it indicates successful initialization.
-#### 2.2 Config application and Run SeaTunnel Web Backend Server
+#### 2.3 Build the project
+
+```shell
+sh build.sh code
+```
+
+#### 2.4 Config application and Run SeaTunnel Web Backend Server
1. Edit `seatunnel-server/seatunnel-app/src/main/resources/application.yml` Fill in the database connection information
+2. Edit `seatunnel-server/seatunnel-app/src/main/resources/application.yml`, add `jwt.secretKey` value. Eg: SeaTunnel.
![img.png](docs/images/application_config.png)
+2. Copy `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` file to `seatunnel-web/seatunnel-server/seatunnel-app/src/main/resources` dir.
+3. Run `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` If there are no errors reported, the seatunnel web backend service is successfully started. Notice that, you must set `-DSEATUNNEL_HOME=${your_seatunnel_install_path}` like this:
+
+![img.png](docs/images/idea_st_home.png)
-2. Run `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` If there are no errors reported, the seatunnel web backend service is successfully started.
+Because the data source plugin is dynamically loaded, it is necessary to set relevant environment variables:
+
+![img.png](docs/images/st_web_basedir_path.png)
#### 2.3 Run SeaTunnel Web Front End
@@ -88,17 +110,46 @@ If there are no issues with the operation, the following information will be dis
Accessing in a browser http://127.0.0.1:5173/login Okay, the default username and password are admin/admin.
### 3 Run SeaTunnel Web In Server
+To run SeaTunnel Web on the server, you need to first have a SeaTunnel Zeta Engine Server environment. If you do not already have one, you can refer to the following steps for deployment.
+
+#### 3.1 Deploy SeaTunnel Zeta Engine Server In Server Node
+
+You have two ways to get the SeaTunnel installer package. Build from source code or download from the SeaTunnel website.
+
+**The SeaTunnel version used here is only for writing this document to show you the process used, and does not necessarily represent the correct version. SeaTunnel Web and SeaTunnel Engine have strict version dependencies, and you can confirm the specific version mapping through xxx**
+
+##### 3.1.1 Build from source code
+* Get the source package from https://seatunnel.apache.org/download or https://github.com/apache/seatunnel.git
+* Build installer package use maven command `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" `
+* Then you can get the installer package in `${Your_code_dir}/seatunnel-dist/target`, For example:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`
+
+##### 3.1.2 Download installer package
+The other way to get SeaTunnel Zeta Engine Server installer package is download the installer package from https://seatunnel.apache.org/download and install plugins online.
-#### 3.1 Build Install Package From Code
+* Download and install connector plugin(Some third-party dependency packages will also be automatically downloaded and installed during this process, such as hadoop jar). You can get the step from https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment.
+* After completing the previous step, you will receive an installation package that can be used to install SeaTunnel Zeta Engine Server on the server. Run `tar -zcvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz apache-seatunnel-2.3.3-SNAPSHOT`
+
+##### 3.1.3 Deploy SeaTunnel Zeta Server
+After 3.1.1 or 3.1.2 you can get an installer package `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`, Then you can copy it to you server node and deploy reference https://seatunnel.apache.org/docs/seatunnel-engine/deployment.
+
+##### 3.1.4 Deploy SeaTunnel Zeta Client In SeaTunnel Web Run Node
+If you use SeaTunnel Web, you need deploy a SeaTunnel Zeta Client in the SeaTunnel Web run Node. **If you run SeaTunnel Zeta Server and SeaTunnel Web in same node, you can skip this step**.
+
+* Copy `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` to the SeaTunnel Web node and unzip it **in the same path of SeaTunnel Zeta Server node**.
+* Set `SEATUNNEL_HOME` to environment variable like SeaTunnel Zeta Server node.
+* Config `hazelcast-client.yaml` reference https://seatunnel.apache.org/docs/seatunnel-engine/deployment#6-config-seatunnel-engine-client
+* Run `$SEATUNNEL_HOME/bin/seatunnel.sh --config $SEATUNNEL_HOME/config/v2.batch.config.template`, If this job run finished, it indicates successful client deployment.
+
+#### 3.2 Build SeaTunnel Web Install Package From Code
```
-cd incubator-seatunnel-web
+cd seatunnel-web
sh build.sh code
```
-Then you can find the installer package in dir `incubator-seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz`.
+Then you can find the installer package in dir `seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz`.
-#### 3.2 Install
+#### 3.3 Install
Copy the `apache-seatunnel-web-${project.version}.tar.gz` to your server node and unzip it.
@@ -106,7 +157,7 @@ Copy the `apache-seatunnel-web-${project.version}.tar.gz` to your server node an
tar -zxvf apache-seatunnel-web-${project.version}.tar.gz
```
-#### 3.3 Init database
+#### 3.4 Init database
1. Edit `apache-seatunnel-web-${project.version}/script/seatunnel_server_env.sh` file, Complete the installed database address, port, username, and password. Here is an example:
@@ -118,13 +169,16 @@ tar -zxvf apache-seatunnel-web-${project.version}.tar.gz
```
2. Run init shell `sh apache-seatunnel-web-${project.version}/script/init_sql.sh` If there are no errors during operation, it indicates successful initialization.
-#### 3.4 Config application and Run SeaTunnel Web Backend Server
+#### 3.5 Config application and Run SeaTunnel Web Backend Server
-Edit `apache-seatunnel-web-${project.version}/config/application.yml` Fill in the database connection information and DS interface related information in the file.
+* Edit `apache-seatunnel-web-${project.version}/conf/application.yml` Fill in the database connection information and DS interface related information in the file.
![image](docs/images/application_config.png)
-#### 3.5 Start SeaTunnel Web
+* Copy `$SEATUNNEL_HOME/config/hazelcast-client.yaml` to `apache-seatunnel-web-${project.version}/conf/`
+* Copy `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` file to `apache-seatunnel-web-${project.version}/conf/` dir.
+
+#### 3.6 Start SeaTunnel Web
```shell
cd apache-seatunnel-web-${project.version}
diff --git a/README_CN.md b/README_CN.md
new file mode 100644
index 000000000..1e3bd9cfe
--- /dev/null
+++ b/README_CN.md
@@ -0,0 +1,208 @@
+# Apache SeaTunnel
+
+
+
+[![Backend Workflow](https://github.com/apache/seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/seatunnel/actions/workflows/backend.yml)
+[![Slack](https://img.shields.io/badge/slack-%23seatunnel-4f8eba?logo=slack)](https://join.slack.com/t/apacheseatunnel/shared_invite/zt-123jmewxe-RjB_DW3M3gV~xL91pZ0oVQ)
+[![Twitter Follow](https://img.shields.io/twitter/follow/ASFSeaTunnel.svg?label=Follow&logo=twitter)](https://twitter.com/ASFSeaTunnel)
+
+---
+[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
+
+SeaTunnel的前身是Waterdrop,并于2021年10月12日更名为SeaTunnel。
+
+---
+
+那么,SeaTunnel是什么?
+SeaTunnel是下一代超高性能、分布式、海量数据集成工具。它每天可以稳定高效地同步数百亿个数据,并已在许多公司的生产中使用。SeaTunnel-web为SeaTunnel提供可视化界面,方便用户使用Apache SeaTunnel。
+
+如果你想知道更多关于SeaTunnel的设计。 👉🏻[设计](https://github.com/apache/seatunnel/issues/1947)
+
+
+## 如何开始
+
+注意:一些细节请参考 docs/QuickStart.md
+
+### 1 准备Apache SeaTunnel环境
+
+因为SeaTunnel Web使用SeaTunnel Java客户端来提交作业,所以运行SeaTunnel Web需要首先准备SeaTunnel Zeta Engine服务。
+
+根据SeaTunnel Zeta Engine的使用要求,提交作业的SeaTunnel Client节点必须与运行作业的SeaTunnel Server节点具有相同的操作系统和安装目录结构。比如如果你想在本地IDEA中运行SeaTunnel Web,你必须在与IDEA相同的机器上安装和运行SeaTunnel Zeta引擎服务器。
+
+别担心,接下来的步骤将告诉您如何在不同情况下正确安装SeaTunnel Zeta Engine Server。
+### 2 在IDEA中运行SeaTunnel Web
+
+如果您想部署和运行SeaTunnel Web,请转到 [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server)
+
+#### 2.1 安装SeaTunnel Zeta引擎服务器
+有两种方法可以获得SeaTunnel安装包。从源代码构建或从SeaTunnel网站下载。
+
+**此处使用的SeaTunnel版本仅用于编写本文档以向您展示所使用的过程,并不一定代表正确的版本。SeaTunnel Web和SeaTunnel Engine有严格的版本依赖关系,您可以通过确认具体的版本对应关系。现在只支持在本地构建SeaTunnel Web和SeaTunnel Zeta Engine,因为有必要确保SeaTunnel Web中的SeaTunnel-api与SeaTunnel Zeta Engine中的版本相同**
+##### 2.1.1 从源代码构建和部署
+* 从https://seatunnel.apache.org/download 或 https://github.com/apache/seatunnel.git 获取源包
+* 构建安装包使用maven命令。`./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" `
+* 在构建之后,需要设置一个环境变量`ST_WEB_BASEDIR_PATH`来表示数据源shade包的位置。将使用自定义类加载器来基于此加载数据源shade包。例如:`ST_WEB_BASEDIR_PATH=/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/`
+然后你可以在`${Your_code_dir}/seatunnel-dist/target`下获取安装包,例如:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`。
+* 执行`tar -zxvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`解压安装包。
+* 运行`cd apache-seatunnel-2.3.3-SNAPSHOT & sh bin/seatunnel-cluster.sh -d`运行SeaTunnel Zeta Engine Server。
+* SeaTunnel Zeta Engine Server默认占用5801端口,请确认端口5801正在被SeaTunnelServer进程占用。
+
+##### 2.1.2 下载安装程序包并进行部署
+下载安装包并部署SeaTunnel Zeta Engine Server的另一种安装方式是从https://seatunnel.apache.org/download 下载安装包并部署。
+
+* 下载并安装连接器插件(一些第三方依赖包也会在此过程中自动下载并安装,如Hadoop jar)。您可以从https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment 获得该步骤。
+* 运行`cd apache-seatunnel-2.3.2 & sh bin/seatunnel-cluster.sh -d`运行seattunnel Zeta Engine Server。
+#### 2.2 初始化数据库
+
+1. 编辑 `seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_env.sh` 文件, 填写已安装的数据库 address, port, username, and password. 下面是一个例子:
+
+ ```
+ export HOSTNAME="localhost"
+ export PORT="3306"
+ export USERNAME="root"
+ export PASSWORD="123456"
+ ```
+2. 执行命令 `sh seatunnel-server/seatunnel-app/src/main/resources/script/init_sql.sh` 如果运行过程中没有错误,则说明初始化成功。
+
+#### 2.3 构建项目
+
+```shell
+sh build.sh code
+```
+
+#### 2.4 配置应用程序并运行SeaTunnel Web后端服务器
+
+1. 编辑 `seatunnel-server/seatunnel-app/src/main/resources/application.yml` 写数据库连接信息
+
+![img.png](docs/images/application_config.png)
+
+2. 复制 `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` 文件 到 `seatunnel-web/seatunnel-server/seatunnel-app/src/main/resources` 目录.
+3. 运行 `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` 如果没有报错,说明seatunnel web后端服务启动成功。注意,你必须设置 `-DSEATUNNEL_HOME=${your_seatunnel_install_path}` 像这样:
+
+![img.png](docs/images/idea_st_home.png)
+
+由于数据源插件是动态加载的,所以需要设置相关的环境变量:
+
+![img.png](docs/images/st_web_basedir_path.png)
+
+#### 2.3 运行SeaTunnel Web Front End
+
+```
+cd seatunnel-ui
+npm install
+npm run dev
+
+```
+
+如果操作正常,系统显示如下信息:
+
+```
+ ➜ Local: http://127.0.0.1:5173/
+ ➜ Network: use --host to expose
+ ➜ press h to show help
+
+```
+
+在浏览器中访问 `http://127.0.0.1:5173/login` , 默认用户名和密码是 `admin/admin`
+
+### 3 在服务器上运行 SeaTunnel Web
+要在服务器上运行SeaTunnel Web,首先需要有一个SeaTunnel Zeta引擎服务器环境。如果还没有,可以参考以下步骤进行部署。
+
+#### 3.1 在Server Node中部署SeaTunnel Zeta Engine Server
+
+有两种方法可以获得SeaTunnel安装包。从源代码构建或从SeaTunnel网站下载。
+
+**此处使用的SeaTunnel版本仅用于编写本文档以向您展示所使用的过程,并不一定代表正确的版本。SeaTunnel Web和SeaTunnel Engine有严格的版本依赖关系,您可以通过xxx确认具体的版本对应关系**
+
+##### 3.1.1 从源码编译
+* 从 https://seatunnel.apache.org/download 或 https://github.com/apache/seatunnel.git 获取源码包
+* 使用maven命令编译安装包 `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" `
+* 然后您可以在`${您的代码目录}/seatunnel-dist/target`中获得安装包,例如:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`
+
+##### 3.1.2 下载安装包
+获取 SeaTunnel Zeta 引擎服务安装包的另一种方式是从 https://seatunnel.apache.org/download 下载安装包并在线安装插件。
+
+* 下载并安装连接器插件(这个过程中也会自动下载和安装一些第三方依赖包,比如 hadoop jar)。您可以参考 https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment 获取详细步骤。
+* 完成上述步骤后,您将获得一个可以用于在服务器上安装 SeaTunnel Zeta 引擎服务的安装包。运行 `tar -zcvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz apache-seatunnel-2.3.3-SNAPSHOT`
+
+##### 3.1.3 部署 SeaTunnel Zeta 服务端
+
+在完成 3.1.1 或 3.1.2 后您可以获得一个安装包 `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`,然后您可以将其复制到服务器节点并参考 https://seatunnel.apache.org/docs/seatunnel-engine/deployment 进行部署。
+
+##### 3.1.4 在 SeaTunnel Web 运行节点部署 SeaTunnel Zeta 客户端
+
+如果您使用 SeaTunnel Web,您需要在 SeaTunnel Web 运行节点部署一个 SeaTunnel Zeta 客户端。**如果 SeaTunnel Zeta 服务端和 SeaTunnel Web 在同一节点,您可以跳过此步骤**。
+
+* 将 `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` 复制到 SeaTunnel Web 节点并解压,解压**到与 SeaTunnel Zeta 服务端节点相同的路径下**。
+* 和 SeaTunnel Zeta 服务端节点相同,设置 `SEATUNNEL_HOME` 环境变量。
+* 参考 https://seatunnel.apache.org/docs/seatunnel-engine/deployment#6-config-seatunnel-engine-client 配置 `hazelcast-client.yaml`。
+* 运行 `$SEATUNNEL_HOME/bin/seatunnel.sh --config $SEATUNNEL_HOME/config/v2.batch.config.template`,如果该作业运行结束,表示客户端部署成功。
+
+#### 3.2 从源码构建 SeaTunnel Web 安装包
+
+```
+cd seatunnel-web
+sh build.sh code
+```
+
+然后您可以在 `seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz` 目录中获得安装包。
+
+#### 3.3 安装
+
+将 `apache-seatunnel-web-${project.version}.tar.gz` 复制到服务器节点并解压。
+
+```shell
+tar -zxvf apache-seatunnel-web-${project.version}.tar.gz
+```shell
+tar -zxvf apache-seatunnel-web-${project.version}.tar.gz
+```
+
+#### 3.4 初始化数据库
+
+1. 编辑 `apache-seatunnel-web-${project.version}/script/seatunnel_server_env.sh` 文件, 填写已安装的数据库 address, port, username, and password. 例如:
+
+ ```
+ export HOSTNAME="localhost"
+ export PORT="3306"
+ export USERNAME="root"
+ export PASSWORD="123456"
+ ```
+2. 运行初始化脚本 `sh apache-seatunnel-web-${project.version}/script/init_sql.sh` 如果操作过程中没有错误,表示初始化成功。
+
+#### 3.5 配置应用并运行 SeaTunnel Web 后端服务
+
+* 编辑 `apache-seatunnel-web-${project.version}/conf/application.yml` 在文件中填写数据库连接信息和数据服务接口相关信息。
+* 编辑 `apache-seatunnel-web-${project.version}/conf/application.yml` 文件,填写jwt.secretKey密钥,例如:SeaTunnel。
+
+![image](docs/images/application_config.png)
+
+* 复制 `$SEATUNNEL_HOME/config/hazelcast-client.yaml` 到 `apache-seatunnel-web-${project.version}/conf/`
+* 复制 `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` 文件到 `apache-seatunnel-web-${project.version}/conf/` 目录.
+
+#### 3.6 运行 SeaTunnel Web
+
+```shell
+cd apache-seatunnel-web-${project.version}
+sh bin/seatunnel-backend-daemon.sh start
+```
+
+在浏览器中访问 http://127.0.0.1:8801/ui/ , 默认用户名和密码是 admin/admin.
+
+### 如何使用
+
+完成所有前置工作后,我们可以打开以下网址: http://127.0.0.1:7890(请根据您的配置替换)来使用它。
+
+现在,让我向您展示如何使用它。
+
+#### 用户管理
+
+![img.png](docs/images/UserImage.png)
+
+#### 任务管理
+![img.png](docs/images/TaskImage.png)
+
+#### 数据源管理
+![img.png](docs/images/DatasourceImage.png)
+
+#### 虚拟表
+![img.png](docs/images/VirtualImage.png)
diff --git a/build.sh b/build.sh
index da4f8b1df..cf2e47ae2 100644
--- a/build.sh
+++ b/build.sh
@@ -23,18 +23,22 @@ WORKDIR=$(
pwd
)
-DOCKER_VERSION=1.0.0-snapshot
+if [ -z "$2" ]; then
+ VERSION="latest"
+else
+ VERSION=$2
+fi
# build code
code() {
- /bin/sh $WORKDIR/mvnw clean package -DskipTests
+ /bin/sh $WORKDIR/mvnw clean package -DskipTests -Pci
# mv release zip
- mv $WORKDIR/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT.zip $WORKDIR/
+ mv $WORKDIR/seatunnel-web-dist/target/apache-seatunnel-web-*.zip $WORKDIR/
}
# build image
image() {
- docker buildx build --load --no-cache -t apache/seatunnel-web:$DOCKER_VERSION -t apache/seatunnel-web:latest -f $WORKDIR/docker/backend.dockerfile .
+ docker buildx build --load --no-cache -t apache/seatunnel-web:$VERSION -t apache/seatunnel-web:latest -f $WORKDIR/docker/backend.dockerfile .
}
# main
@@ -46,7 +50,7 @@ case "$1" in
image
;;
*)
- echo "Usage: seatunnel-daemon.sh {start|stop|status}"
+ echo "Usage: build.sh {code|image}"
exit 1
;;
esac
diff --git a/docs/QuickStart.md b/docs/QuickStart.md
deleted file mode 100644
index 79dc6f485..000000000
--- a/docs/QuickStart.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# 1.SeaTunnel 配置
-
-## 1.1拉取SeaTunnel代码
-
-github地址: https://github.com/apache/seatunnel
-
-## 1.2Install SeaTunnel
-
-1. 检查SeaTunnel version ,在根目录的pom文件中
-
- ![img.png](images\image-20230727135201238.png)
-
-2. 在终端中输入命令 : ./mvnw clean install '-Dmaven.test.skip=true' '-Dcheckstyle.skip=true 或者输入 mvn clean install -DskipTests=true
-
-3. 执行完成后,显示如下:
-
-![img.png](images/image-20230727134919480.png)
-
-## 1.3启动SeaTunnel引擎
-
-执行seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java文件,启动SeaTunnel引擎服务
-
-# 2.SeaTunnel Web设置
-
-## 2.1SeaTunnel依赖版本设置
-
-在ST-Web跟目录的pom文件中设置SeaTunnel引擎版本,参考1.2中的SeaTunnel version
-
- ![img.png](images\image-20230727141624507.png)
-
-## 2.2添加 Connector依赖
-
-假如你想使用Kafka作为数据源,需要在两个地方添加Kafka连接器的依赖
-
-1. 在根目录的pom文件中的中添加kafka连接器依赖管理(注意修改该依赖的(不要是test级别))
-
- ![img.png](images\image-20230727144108242.png)
-
-2. 在seatunnel-server/seatunnel-app/pom.xml文件中添加kafka连接器依赖(注意修改该依赖的(不要是test级别))
-
- ![img.png](images\image-20230727144320362.png)
-
-## 2.3 打包SeaTunnel Web 项目
-
-使用命令 mvn clean install -DskipTests=true
-
-## 2.4添加Enviroment variables
-
-1. 当打包完成之后,在文件seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/apache-seatunnel-web-1.0.0-SNAPSHOT(在第三步中会用到)
-
-2. 假如你使用的是IDEA,打开Edit Configurations
-
- ![img.png](images\image-20230727145045703.png)
-
-3. 添加Enviroment variables,将apache-seatunnel-web-1.0.0-SNAPSHOT文件的绝对路径添加的Enviroment variables中,命名为ST_WEB_BASEDIR_PATH
-
-![img.png](images\image-20230727145203278.png)
-
- ![img.png](images\image-20230727145452333.png)
-
-![image-20230727145652824](images\image-20230727145652824.png)
-
-4. 在SeatunnelApplication.java 下启动ST-Web
-
-# 3.QuickStart
-
-打开链接: http://127.0.0.1:5173/login
-
-初始用户名和密码都是admin
-
- ![img.png](images\image-20230727163613305.png)
-
-## 3.1创建数据源
-
- ![image-20230727163756006](images\image-20230727163756006.png)
-
-选择一个你需要使用的数据源
-
-![img.png](images\image-20230727163903835.png)
-
-![img.png](images\image-20230727164019922.png)
-
-## 3.2创建同步任务
-
-![image-20230727164350427](images\image-20230727164350427.png)
-
diff --git a/docs/images/ds_create_project.png b/docs/images/ds_create_project.png
deleted file mode 100644
index 71bb25168..000000000
Binary files a/docs/images/ds_create_project.png and /dev/null differ
diff --git a/docs/images/ds_create_tenant.png b/docs/images/ds_create_tenant.png
deleted file mode 100644
index 45dfe3363..000000000
Binary files a/docs/images/ds_create_tenant.png and /dev/null differ
diff --git a/docs/images/ds_create_token.png b/docs/images/ds_create_token.png
deleted file mode 100644
index 539c0bdf1..000000000
Binary files a/docs/images/ds_create_token.png and /dev/null differ
diff --git a/docs/images/ds_create_user.png b/docs/images/ds_create_user.png
deleted file mode 100644
index 5117f398b..000000000
Binary files a/docs/images/ds_create_user.png and /dev/null differ
diff --git a/docs/images/idea_st_home.png b/docs/images/idea_st_home.png
new file mode 100644
index 000000000..d4fd83493
Binary files /dev/null and b/docs/images/idea_st_home.png differ
diff --git a/docs/images/image-20230727134919480.png b/docs/images/image-20230727134919480.png
deleted file mode 100644
index 0851b5a30..000000000
Binary files a/docs/images/image-20230727134919480.png and /dev/null differ
diff --git a/docs/images/image-20230727135201238.png b/docs/images/image-20230727135201238.png
deleted file mode 100644
index 67ea06103..000000000
Binary files a/docs/images/image-20230727135201238.png and /dev/null differ
diff --git a/docs/images/image-20230727135931257.png b/docs/images/image-20230727135931257.png
deleted file mode 100644
index 2b08b0035..000000000
Binary files a/docs/images/image-20230727135931257.png and /dev/null differ
diff --git a/docs/images/image-20230727140031522.png b/docs/images/image-20230727140031522.png
deleted file mode 100644
index 9e5a708a8..000000000
Binary files a/docs/images/image-20230727140031522.png and /dev/null differ
diff --git a/docs/images/image-20230727141624507.png b/docs/images/image-20230727141624507.png
deleted file mode 100644
index 52a75a6fd..000000000
Binary files a/docs/images/image-20230727141624507.png and /dev/null differ
diff --git a/docs/images/image-20230727144108242.png b/docs/images/image-20230727144108242.png
deleted file mode 100644
index 261b948b6..000000000
Binary files a/docs/images/image-20230727144108242.png and /dev/null differ
diff --git a/docs/images/image-20230727144320362.png b/docs/images/image-20230727144320362.png
deleted file mode 100644
index df2a9177f..000000000
Binary files a/docs/images/image-20230727144320362.png and /dev/null differ
diff --git a/docs/images/image-20230727145045703.png b/docs/images/image-20230727145045703.png
deleted file mode 100644
index 2db92758d..000000000
Binary files a/docs/images/image-20230727145045703.png and /dev/null differ
diff --git a/docs/images/image-20230727145203278.png b/docs/images/image-20230727145203278.png
deleted file mode 100644
index 2cc4bfad0..000000000
Binary files a/docs/images/image-20230727145203278.png and /dev/null differ
diff --git a/docs/images/image-20230727145452333.png b/docs/images/image-20230727145452333.png
deleted file mode 100644
index d9558c764..000000000
Binary files a/docs/images/image-20230727145452333.png and /dev/null differ
diff --git a/docs/images/image-20230727145652824.png b/docs/images/image-20230727145652824.png
deleted file mode 100644
index cd48cf78e..000000000
Binary files a/docs/images/image-20230727145652824.png and /dev/null differ
diff --git a/docs/images/image-20230727163613305.png b/docs/images/image-20230727163613305.png
deleted file mode 100644
index c86296bc8..000000000
Binary files a/docs/images/image-20230727163613305.png and /dev/null differ
diff --git a/docs/images/image-20230727163756006.png b/docs/images/image-20230727163756006.png
deleted file mode 100644
index 4ddbdb4e1..000000000
Binary files a/docs/images/image-20230727163756006.png and /dev/null differ
diff --git a/docs/images/image-20230727163903835.png b/docs/images/image-20230727163903835.png
deleted file mode 100644
index 599d0c6fb..000000000
Binary files a/docs/images/image-20230727163903835.png and /dev/null differ
diff --git a/docs/images/image-20230727164019922.png b/docs/images/image-20230727164019922.png
deleted file mode 100644
index fc36bfa77..000000000
Binary files a/docs/images/image-20230727164019922.png and /dev/null differ
diff --git a/docs/images/image-20230727164350427.png b/docs/images/image-20230727164350427.png
deleted file mode 100644
index e77dd021b..000000000
Binary files a/docs/images/image-20230727164350427.png and /dev/null differ
diff --git a/docs/images/st_web_basedir_path.png b/docs/images/st_web_basedir_path.png
new file mode 100644
index 000000000..fff06c12b
Binary files /dev/null and b/docs/images/st_web_basedir_path.png differ
diff --git a/pom.xml b/pom.xml
index adcde5485..c0e49c738 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,7 +61,7 @@
3.3.0
false
- 2.12.7.1
+ 2.12.6
2.12.7
8.0.16
2.1.214
@@ -71,11 +71,9 @@
19.0
3.10.0
4.2.0
- 2.3.2
+ 2.3.3
21.5.0.0
- 0.3.2-patch11
42.4.3
- 2.1.0.10
9.2.1.jre8
8.0.28
3.1.4
@@ -111,6 +109,7 @@
1.2.17
2.17.1
1.7.25
+ false
@@ -256,6 +255,12 @@
org.apache.seatunnel
seatunnel-common
${seatunnel-framework.version}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -275,6 +280,12 @@
org.apache.seatunnel
seatunnel-api
${seatunnel-framework.version}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -294,6 +305,12 @@
org.apache.seatunnel
seatunnel-plugin-discovery
${seatunnel-framework.version}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -412,33 +429,31 @@
${jsoup.version}
-
+
- org.apache.seatunnel
- connector-common
- ${seatunnel-framework.version}
- test
+ cn.hutool
+ hutool-all
+ 5.8.16
- org.apache.seatunnel
- connector-console
- ${seatunnel-framework.version}
- test
+ com.h2database
+ h2
+ ${h2.version}
+
org.apache.seatunnel
- connector-fake
+ connector-common
${seatunnel-framework.version}
test
-
-
-
- org.apache.seatunnel
- connector-jdbc
- ${seatunnel-framework.version}
-
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -482,17 +497,11 @@
${checker.qual.version}
-
- org.awaitility
- awaitility
- ${awaitility.version}
- test
-
-
org.apache.seatunnel
seatunnel-hadoop3-3.1.4-uber
${hadoop-uber.version}
+ provided
org.apache.avro
@@ -545,7 +554,7 @@
org.apache.seatunnel
datasource-s3
${project.version}
- provided
+ test
@@ -556,23 +565,23 @@
- org.apache.seatunnel
- connector-common
- ${seatunnel-framework.version}
+ org.awaitility
+ awaitility
+ ${awaitility.version}
test
-
- org.apache.seatunnel
- seatunnel-transforms-v2
- ${seatunnel-framework.version}
-
-
org.apache.seatunnel
connector-console
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -580,300 +589,602 @@
connector-fake
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-kafka
${seatunnel-framework.version}
+ test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-base
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-feishu
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-wechat
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-myhours
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-lemlist
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-klaviyo
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-onesignal
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-notion
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-jdbc
${seatunnel-framework.version}
-
+ test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-socket
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-clickhouse
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-pulsar
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-hive
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-hadoop
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-local
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-oss
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-oss-jindo
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-ftp
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-sftp
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-hudi
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-dingtalk
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-kudu
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-email
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-elasticsearch
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-iotdb
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+
org.apache.seatunnel
connector-neo4j
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-redis
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-google-sheets
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-datahub
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-sentry
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-mongodb
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-iceberg
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-influxdb
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-cassandra
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-file-s3
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-amazondynamodb
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-starrocks
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-tablestore
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-slack
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-gitlab
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-http-jira
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-rabbitmq
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-openmldb
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-doris
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-maxcompute
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-cdc-mysql
${seatunnel-framework.version}
-
+ test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-cdc-sqlserver
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
org.apache.seatunnel
connector-cdc-oracle
${seatunnel-framework.version}
test
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
@@ -885,18 +1196,6 @@
commons-lang3
${commons-lang3.version}
-
- org.apache.seatunnel
- connector-cdc-sqlserver
- ${seatunnel-framework.version}
- test
-
-
- org.apache.seatunnel
- connector-cdc-mysql
- ${seatunnel-framework.version}
-
-
org.projectlombok
lombok
@@ -1134,6 +1433,7 @@
spotless-maven-plugin
${spotless.version}
+ ${skip.spotless}
1.7
@@ -1217,7 +1517,6 @@
org.apache.maven.plugins
maven-release-plugin
- 3.0.1
true
@{project.version}
@@ -1237,11 +1536,6 @@
maven-surefire-plugin
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
org.codehaus.mojo
license-maven-plugin
@@ -1254,7 +1548,7 @@
- https://github.com/apache/incubator-seatunnel
+ https://github.com/apache/seatunnel-web
@@ -1279,15 +1573,15 @@
- scm:git:https://github.com/apache/incubator-seatunnel.git
- scm:git:https://github.com/apache/incubator-seatunnel.git
- https://github.com/apache/incubator-seatunnel
+ scm:git:https://github.com/apache/seatunnel-web.git
+ scm:git:https://github.com/apache/seatunnel-web.git
+ https://github.com/apache/seatunnel-web
HEAD
GitHub
- https://github.com/apache/incubator-seatunnel/issues
+ https://github.com/apache/seatunnel/issues
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java
index 9c629859f..34fb2850e 100644
--- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java
@@ -67,11 +67,11 @@ protected AbstractDataSourceClient() {
for (String pluginName : DatasourceLoadConfig.pluginSet) {
log.info("plugin set : " + pluginName);
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(getCustomClassloader(pluginName));
if (DatasourceLoadConfig.classLoaderChannel.get(pluginName.toUpperCase()) != null) {
log.info(pluginName + " is exist");
continue;
}
+ Thread.currentThread().setContextClassLoader(getCustomClassloader(pluginName));
try {
Class> clazz =
Class.forName(
@@ -81,7 +81,7 @@ protected AbstractDataSourceClient() {
Thread.currentThread().getContextClassLoader());
DataSourceFactory factory =
(DataSourceFactory) clazz.getDeclaredConstructor().newInstance();
- log.info("factory : " + String.valueOf(factory));
+ log.info("factory : " + factory);
Set dataSourcePluginInfos = factory.supportedDataSources();
dataSourcePluginInfos.forEach(
dataSourceInfo -> {
@@ -126,11 +126,7 @@ public List listAllDataSources() {
protected DataSourceChannel getDataSourceChannel(String pluginName) {
checkNotNull(pluginName, "pluginName cannot be null");
- // Integer index = supportedDataSourceIndex.get(pluginName.toUpperCase());
- // if (index == null) {
- // throw new DataSourceSDKException(
- // "The %s plugin is not supported or plugin not exist.", pluginName);
- // }
+
return DatasourceLoadConfig.classLoaderChannel.get(pluginName.toUpperCase());
}
@@ -229,7 +225,6 @@ private ClassLoader getCustomClassloader(String pluginName) {
log.info("ST_WEB_BASEDIR_PATH is : " + getenv);
String libPath = StringUtils.isEmpty(getenv) ? "/datasource" : (getenv + "/datasource");
- // String libPath = "/root/apache-seatunnel-web-2.4.7-WS-SNAPSHOT/datasource/";
File jarDirectory = new File(libPath);
File[] jarFiles =
jarDirectory.listFiles(
@@ -249,7 +244,11 @@ private ClassLoader getCustomClassloader(String pluginName) {
});
log.info("jar file length :" + (jarFiles == null ? 0 : jarFiles.length));
- log.info("jar file length :" + (jarFiles == null ? 0 : jarFiles[0].getName()));
+ log.info(
+ "jar file name :"
+ + (jarFiles == null
+ ? 0
+ : jarFiles.length == 0 ? "no jar" : jarFiles[0].getName()));
DatasourceClassLoader customClassLoader =
DatasourceLoadConfig.datasourceClassLoaders.get(pluginName.toUpperCase());
try {
@@ -272,7 +271,7 @@ private ClassLoader getCustomClassloader(String pluginName) {
} catch (Exception e) {
e.printStackTrace();
}
- log.info("custom loader is:" + String.valueOf(customClassLoader));
+ log.info("custom loader is:" + customClassLoader);
return customClassLoader;
}
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java
index f29f29c9f..9ef3f3a7f 100644
--- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java
@@ -65,7 +65,6 @@ public Class> loadClass(String name) throws ClassNotFoundException {
@Override
public URL getResource(String name) {
- log.info("getResource : " + name);
// first, try and find it via the URLClassloader
URL urlClassLoaderResource = findResource(name);
if (urlClassLoaderResource != null) {
@@ -79,7 +78,6 @@ public URL getResource(String name) {
@Override
public Enumeration getResources(String name) throws IOException {
// first get resources from URLClassloader
- log.info("getResources : " + name);
Enumeration urlClassLoaderResources = findResources(name);
final List result = new ArrayList<>();
diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
index d169c310b..b2869691a 100644
--- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
+++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java
@@ -57,8 +57,7 @@ public class DatasourceLoadConfig {
"JDBC-CLICKHOUSE",
"org.apache.seatunnel.datasource.plugin.clickhouse.jdbc.ClickhouseJdbcDataSourceFactory");
classLoaderFactoryName.put(
- "HIVE",
- "org.apache.seatunnel.datasource.plugin.hive.jdbc.HiveJdbcDataSourceFactory");
+ "HIVE", "com.whaleops.datasource.datasource.plugin.hive.HiveDataSourceFactory");
classLoaderFactoryName.put(
"JDBC-ORACLE",
"org.apache.seatunnel.datasource.plugin.oracle.jdbc.OracleJdbcDataSourceFactory");
@@ -93,6 +92,11 @@ public class DatasourceLoadConfig {
classLoaderFactoryName.put(
"JDBC-STARROCKS",
"org.apache.seatunnel.datasource.plugin.starrocks.jdbc.StarRocksJdbcDataSourceFactory");
+ classLoaderFactoryName.put(
+ "MONGODB", "com.apache.seatunnel.datasource.plugin.mongodb.MongoDataSourceFactory");
+ classLoaderFactoryName.put(
+ "JDBC-DB2",
+ "org.apache.seatunnel.datasource.plugin.db2.jdbc.Db2JdbcDataSourceFactory");
classLoaderJarName.put("JDBC-ORACLE", "datasource-jdbc-oracle-");
classLoaderJarName.put("JDBC-CLICKHOUSE", "datasource-jdbc-clickhouse-");
@@ -107,11 +111,13 @@ public class DatasourceLoadConfig {
classLoaderJarName.put("ELASTICSEARCH", "datasource-elasticsearch-");
classLoaderJarName.put("S3", "datasource-s3-");
- classLoaderJarName.put("HIVE", "datasource-jdbc-hive-");
+ classLoaderJarName.put("HIVE", "datasource-hive-");
classLoaderJarName.put("KAFKA", "datasource-kafka-");
classLoaderJarName.put("STARROCKS", "datasource-starrocks-");
classLoaderJarName.put("S3-REDSHIFT", "datasource-s3redshift-");
classLoaderJarName.put("JDBC-STARROCKS", "datasource-jdbc-starrocks-");
+ classLoaderJarName.put("MONGODB", "datasource-mongodb-");
+ classLoaderJarName.put("JDBC-DB2", "datasource-jdbc-db2-");
}
public static final Set pluginSet =
@@ -122,16 +128,15 @@ public class DatasourceLoadConfig {
"Hive",
"JDBC-Oracle",
"JDBC-Postgres",
- "JDBC-Redshift",
"JDBC-SQLServer",
"JDBC-TiDB",
"Kafka",
"MySQL-CDC",
"S3",
- "S3-Redshift",
"SqlServer-CDC",
- "JDBC-StarRocks",
- "StarRocks");
+ "StarRocks",
+ "MongoDB",
+ "JDBC-Db2");
public static Map datasourceClassLoaders = new HashMap<>();
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml
index 2eac02b34..d5dee4503 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml
@@ -55,11 +55,6 @@
datasource-jdbc-tidb
${project.version}
-
- org.apache.seatunnel
- datasource-jdbc-redshift
- ${project.version}
-
org.apache.seatunnel
datasource-jdbc-sqlserver
@@ -82,11 +77,6 @@
datasource-elasticsearch
${project.version}
-
- org.apache.seatunnel
- datasource-s3redshift
- ${project.version}
-
org.apache.seatunnel
datasource-starrocks
@@ -107,6 +97,11 @@
datasource-sqlserver-cdc
${project.version}
+
+ org.apache.seatunnel
+ datasource-jdbc-db2
+ ${project.version}
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml
index 017a221ad..61b995fde 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml
@@ -61,4 +61,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml
new file mode 100644
index 000000000..672885cc9
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml
@@ -0,0 +1,114 @@
+
+
+
+ 4.0.0
+
+ org.apache.seatunnel
+ seatunnel-datasource-plugins
+ ${revision}
+
+
+ datasource-hive
+
+
+ 3.1.3
+ 24.1-jre
+
+
+
+
+ org.apache.seatunnel
+ datasource-plugins-api
+ ${project.version}
+ provided
+
+
+ org.apache.seatunnel
+ seatunnel-hadoop3-3.1.4-uber
+ 2.3.2
+ optional
+
+
+ org.apache.avro
+ avro
+
+
+
+
+ org.apache.hive
+ hive-exec
+ ${hive.exec.version}
+
+
+ log4j
+ log4j
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+ org.apache.logging.log4j
+ log4j-web
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ org.apache.parquet
+ parquet-hadoop-bundle
+
+
+ jdk.tools
+ jdk.tools
+
+
+ org.pentaho
+ pentaho-aggdesigner-algorithm
+
+
+ org.apache.avro
+ avro
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java
new file mode 100644
index 000000000..e9826c59a
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.whaleops.datasource.datasource.plugin.hive;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class HiveConfig {
+ public static final String PLUGIN_NAME = "Hive";
+
+ public static final DataSourcePluginInfo HIVE_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set HIVE_SYSTEM_DATABASES = Sets.newHashSet();
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder().required(HiveOptionRule.METASTORE_URI).build();
+}
diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java
similarity index 70%
rename from seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java
index 658e59b29..d469b3b1b 100644
--- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java
@@ -15,17 +15,15 @@
* limitations under the License.
*/
-package org.apache.seatunnel.spi.scheduler.dto;
+package com.whaleops.datasource.datasource.plugin.hive;
-import org.apache.seatunnel.spi.scheduler.enums.ExecuteTypeEnum;
+import com.google.common.collect.Sets;
-import lombok.Builder;
-import lombok.Data;
+import java.util.Set;
-@Data
-@Builder
-public class ExecuteDto {
- private JobDto jobDto;
- private ExecuteTypeEnum executeTypeEnum;
- private ComplementDataDto complementDataDto;
+public class HiveConstants {
+
+ public static final Set HIVE_SYSTEM_DATABASES =
+ Sets.newHashSet(
+ "information_schema", "mysql", "performance_schema", "sys", "test", "hivedb");
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java
new file mode 100644
index 000000000..cebdbf8e8
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.whaleops.datasource.datasource.plugin.hive;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.whaleops.datasource.datasource.plugin.hive.client.HiveClient;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class HiveDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return HiveOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return HiveOptionRule.metadataRule();
+ }
+
+ public List getTables(
+ @NonNull String pluginName,
+ Map requestParams,
+ String database,
+ Map options) {
+
+ try (HiveClient hiveClient = HiveClient.createInstance(requestParams)) {
+ String size = options.get("size");
+ Integer sizeInt = StringUtils.isNotEmpty(size) ? Integer.parseInt(size) : 0;
+ return hiveClient.getAllTables(database, options.get("filterName"), sizeInt);
+ }
+ }
+
+ @Override
+ public List getDatabases(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+ try (HiveClient hiveClient = HiveClient.createInstance(requestParams); ) {
+ return hiveClient.getAllDatabases();
+ }
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+ return checkHiveConnectivity(requestParams);
+ }
+
+ @Override
+ public List getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ return getTableFields(requestParams, database, table);
+ }
+
+ @Override
+ public Map> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map requestParams,
+ @NonNull String database,
+ @NonNull List tables) {
+ Map> tableFields = new HashMap<>(tables.size());
+ for (String table : tables) {
+ tableFields.put(table, getTableFields(requestParams, database, table));
+ }
+ return tableFields;
+ }
+
+ protected boolean checkHiveConnectivity(Map requestParams) {
+ try (HiveClient ignored = HiveClient.createInstance(requestParams)) {
+ System.out.println(ignored.getAllDatabases());
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException(
+ "check hive connectivity failed, " + e.getMessage(), e);
+ }
+ }
+
+ protected List getTableFields(
+ Map requestParams, String dbName, String tableName) {
+ try (HiveClient hiveClient = HiveClient.createInstance(requestParams)) {
+ return hiveClient.getFields(dbName, tableName);
+ }
+ }
+
+ private static boolean checkHostConnectable(String host, int port) {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, port), 1000);
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ private boolean isNotSystemDatabase(String pluginName, String dbName) {
+ // FIXME,filters system databases
+ return true;
+ }
+
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+}
diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java
similarity index 52%
rename from seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java
index e4d2f08a5..712364532 100644
--- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java
@@ -15,32 +15,31 @@
* limitations under the License.
*/
-package org.apache.seatunnel.scheduler.dolphinscheduler.enums;
+package com.whaleops.datasource.datasource.plugin.hive;
-import org.apache.seatunnel.server.common.SeatunnelErrorEnum;
-import org.apache.seatunnel.server.common.SeatunnelException;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
-public enum ReleaseStateEnum {
- ONLINE(true),
- OFFLINE(false),
- ;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
- private final boolean publish;
+import java.util.Set;
- ReleaseStateEnum(boolean publish) {
- this.publish = publish;
+@Slf4j
+public class HiveDataSourceFactory implements DataSourceFactory {
+ @Override
+ public String factoryIdentifier() {
+ return "Hive";
}
- public static ReleaseStateEnum parse(String name) {
- for (ReleaseStateEnum value : values()) {
- if (value.name().equalsIgnoreCase(name)) {
- return value;
- }
- }
- throw new SeatunnelException(SeatunnelErrorEnum.NO_SUCH_ELEMENT);
+ @Override
+ public Set supportedDataSources() {
+ return Sets.newHashSet(HiveConfig.HIVE_DATASOURCE_PLUGIN_INFO);
}
- public boolean isPublish() {
- return publish;
+ @Override
+ public DataSourceChannel createChannel() {
+ return new HiveDataSourceChannel();
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
new file mode 100644
index 000000000..dcea7d2fa
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.whaleops.datasource.datasource.plugin.hive;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+public class HiveOptionRule {
+
+ public static final Option METASTORE_URI =
+ Options.key("metastore_uri")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("thrift url, eg:" + "thrift://127.0.0.1:9083");
+
+ public static final Option KERBEROS_PRINCIPAL =
+ Options.key("kerberos_principal")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc kerberos_principal");
+
+ public static final Option KERBEROS_KEYTAB_PATH =
+ Options.key("kerberos_keytab_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc kerberos_keytab_path");
+
+ public static final Option KERBEROS_KRB5_CONF_PATH =
+ Options.key("kerberos_krb5_conf_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc kerberos_keytab_path");
+
+ public static final Option HDFS_SITE_PATH =
+ Options.key("hdfs_site_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc hdfs_site_path");
+
+ public static final Option HIVE_SITE_PATH =
+ Options.key("hive_site_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("jdbc hive_site_path");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(METASTORE_URI)
+ .optional(KERBEROS_PRINCIPAL)
+ .optional(KERBEROS_KRB5_CONF_PATH)
+ .optional(KERBEROS_KEYTAB_PATH)
+ .optional(HDFS_SITE_PATH)
+ .optional(HIVE_SITE_PATH)
+ .build();
+ }
+
+ public static OptionRule metadataRule() {
+ // todo
+ return OptionRule.builder().build();
+ }
+
+ public static final Option TABLE_NAME =
+ Options.key("table").stringType().noDefaultValue().withDescription("hive table");
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
new file mode 100644
index 000000000..92bf0e38c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.whaleops.datasource.datasource.plugin.hive.client;
+
+import org.apache.seatunnel.common.utils.ExceptionUtils;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.thrift.TException;
+
+import com.whaleops.datasource.datasource.plugin.hive.HiveOptionRule;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+@Slf4j
+public class HiveClient implements AutoCloseable {
+
+ private final HiveMetaStoreClient hiveMetaStoreClient;
+
+ public HiveClient(HiveMetaStoreClient hiveMetaStoreClient) {
+ this.hiveMetaStoreClient = hiveMetaStoreClient;
+ }
+
+ public static HiveClient createInstance(Map reqParam) {
+ checkNotNull(
+ reqParam.get(HiveOptionRule.METASTORE_URI.key()),
+ "hive metastore_uri cannot be null");
+ String metastoreUri = reqParam.get(HiveOptionRule.METASTORE_URI.key());
+ String kerberosPrincipal = reqParam.get(HiveOptionRule.KERBEROS_PRINCIPAL.key());
+ String kerberosKrb5ConfPath = reqParam.get(HiveOptionRule.KERBEROS_KRB5_CONF_PATH.key());
+ String kerberosKeytabPath = reqParam.get(HiveOptionRule.KERBEROS_KEYTAB_PATH.key());
+ String hdfsSitePath = reqParam.get(HiveOptionRule.HDFS_SITE_PATH.key());
+ String hiveSitePath = reqParam.get(HiveOptionRule.HIVE_SITE_PATH.key());
+ System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath);
+ System.setProperty("krb.principal", "hadoop");
+ try {
+ if (StringUtils.isNotEmpty(kerberosPrincipal)) {
+ // login Kerberos
+ Configuration configuration = new Configuration();
+ if (StringUtils.isNotEmpty(kerberosPrincipal)) {
+ configuration.addResource(new File(hdfsSitePath).toURI().toURL());
+ }
+ doKerberosAuthentication(configuration, kerberosPrincipal, kerberosKeytabPath);
+ }
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.set("hive.metastore.uris", metastoreUri);
+ if (StringUtils.isNotEmpty(hiveSitePath)) {
+ hiveConf.addResource(new File(hiveSitePath).toURI().toURL());
+ }
+ log.info("hive client conf:{}", hiveConf);
+
+ return new HiveClient(new HiveMetaStoreClient(hiveConf));
+ } catch (Exception e) {
+ String errorMsg =
+ String.format(
+ "Using this hive uris [%s] to initialize "
+ + "hive metastore client instance failed",
+ metastoreUri);
+ log.error(ExceptionUtils.getMessage(e));
+ throw new DataSourcePluginException(errorMsg, e);
+ }
+ }
+
+ /*private static void authKerberos(
+ String kerberosKrb5ConfPath, String kerberosKeytabPath, String kerberosPrincipal)
+ throws IOException {
+ System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath);
+ Configuration configuration = new Configuration();
+ configuration.set("hadoop.security.authentication", "Kerberos");
+ configuration.setBoolean("hadoop.security.authorization", true);
+ UserGroupInformation.setConfiguration(configuration);
+ UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabPath);
+ }*/
+
+ public static void doKerberosAuthentication(
+ Configuration configuration, String principal, String keytabPath) {
+ if (StringUtils.isBlank(principal) || StringUtils.isBlank(keytabPath)) {
+ log.warn(
+ "Principal [{}] or keytabPath [{}] is empty, it will skip kerberos authentication",
+ principal,
+ keytabPath);
+ } else {
+ configuration.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(configuration);
+ try {
+ log.info(
+ "Start Kerberos authentication using principal {} and keytab {}",
+ principal,
+ keytabPath);
+ UserGroupInformation.loginUserFromKeytab(principal, keytabPath);
+ log.info("Kerberos authentication successful");
+ } catch (IOException e) {
+ throw new DataSourcePluginException(
+ "check hive connectivity failed, " + e.getMessage(), e);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (hiveMetaStoreClient != null) {
+ hiveMetaStoreClient.close();
+ }
+ } catch (Exception e) {
+ log.warn("close hive connection error", e);
+ }
+ }
+
+ public List getAllDatabases() {
+ try {
+ return hiveMetaStoreClient.getAllDatabases();
+ } catch (Exception e) {
+ log.error(ExceptionUtils.getMessage(e));
+ throw new DataSourcePluginException("get database names failed", e);
+ }
+ }
+
+ public List getAllTables(String dbName) {
+ return getAllTables(dbName, null, null);
+ }
+
+ public List getAllTables(String dbName, String filterName, Integer size) {
+ try {
+
+ List tables = hiveMetaStoreClient.getTables(dbName, "*");
+ List filteredTables = new ArrayList<>();
+ // hive api whether to support filtering directly
+ if (StringUtils.isNotEmpty(filterName)) {
+ for (String tableName : tables) {
+ if (tableName.contains(filterName)) {
+ filteredTables.add(tableName);
+ }
+ }
+ } else {
+ filteredTables.addAll(tables);
+ }
+ // filter Iceberg table
+ this.filteredIcebergTable(filteredTables, dbName);
+ if (size != null && size > 0) {
+ return filteredTables.subList(0, Math.min(size, filteredTables.size()));
+ }
+ return filteredTables;
+ } catch (Exception e) {
+ log.error(ExceptionUtils.getMessage(e));
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ // 过滤掉 Iceberg 表
+ private void filteredIcebergTable(List filteredTables, String dbName)
+ throws TException {
+ if (CollectionUtils.isNotEmpty(filteredTables)) {
+ Iterator iterator = filteredTables.iterator();
+ while (iterator.hasNext()) {
+ String element = iterator.next();
+ if (isIcebergTable(dbName, element)) {
+ iterator.remove();
+ }
+ }
+ }
+ }
+
+ // 判断是否为 Iceberg 表
+ private boolean isIcebergTable(String dbName, String tableName) throws TException {
+ Table table = hiveMetaStoreClient.getTable(dbName, tableName);
+ Map parameters = table.getParameters();
+ String tableType = parameters.get("table_type");
+ return "ICEBERG".equals(tableType);
+ }
+
+ public List getFields(String dbName, String tableName) {
+ List tableFields = new ArrayList<>();
+ try {
+ List fields = hiveMetaStoreClient.getFields(dbName, tableName);
+ fields.forEach(
+ field -> {
+ TableField tableField = new TableField();
+ String columnName = field.getName();
+ tableField.setPrimaryKey(false);
+ tableField.setName(columnName);
+ String stringType = field.getType();
+ String fieldTypeStart = stringType.split("[(<]")[0];
+ tableField.setType(fieldTypeStart);
+ tableField.setComment(field.getComment());
+ tableFields.add(tableField);
+ });
+ return tableFields;
+ } catch (TException e) {
+ log.error(ExceptionUtils.getMessage(e));
+ throw new DataSourcePluginException("get table fields failed", e);
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml
index dcc91da43..31f722bd5 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml
@@ -55,8 +55,25 @@
com.clickhouse
clickhouse-jdbc
${clickhouse.version}
- provided
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml
similarity index 52%
rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml
index 61b84cb8b..a13423735 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml
@@ -22,7 +22,11 @@
${revision}
- datasource-s3-redshift
+ datasource-jdbc-db2
+
+
+ 8.0.28
+
@@ -31,39 +35,58 @@
${project.version}
provided
+
+ org.apache.commons
+ commons-lang3
+
+
+
+ com.google.auto.service
+ auto-service
+
org.apache.seatunnel
- seatunnel-hadoop3-3.1.4-uber
- 2.4.5-WS-SNAPSHOT
-
-
- org.apache.avro
- avro
-
-
+ seatunnel-api
+ provided
+
+
- com.amazon.redshift
- redshift-jdbc42
- ${redshift.version}
+ mysql
+ mysql-connector-java
+ ${mysql-connector.version}
provided
+
- org.apache.hadoop
- hadoop-aws
- ${hadoop-aws.version}
-
-
- jdk.tools
- jdk.tools
-
-
+ com.ibm.db2.jcc
+ db2jcc
+ db2jcc4
- com.amazonaws
- aws-java-sdk-bundle
- ${aws-java-sdk-bundle.version}
+ org.apache.seatunnel
+ seatunnel-api
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java
new file mode 100644
index 000000000..90673587c
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.db2.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
+import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
+
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
+public class Db2DataSourceConfig {
+
+ public static final String PLUGIN_NAME = "JDBC-Db2";
+
+ public static final DataSourcePluginInfo DB2_DATASOURCE_PLUGIN_INFO =
+ DataSourcePluginInfo.builder()
+ .name(PLUGIN_NAME)
+ .icon(PLUGIN_NAME)
+ .version("1.0.0")
+ .type(DatasourcePluginTypeEnum.DATABASE.getCode())
+ .build();
+
+ public static final Set DB2_SYSTEM_DATABASES =
+ Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys");
+
+ public static final OptionRule OPTION_RULE =
+ OptionRule.builder()
+ .required(Db2OptionRule.URL, Db2OptionRule.DRIVER)
+ .optional(Db2OptionRule.USER, Db2OptionRule.PASSWORD)
+ .build();
+
+ public static final OptionRule METADATA_RULE =
+ OptionRule.builder().required(Db2OptionRule.DATABASE, Db2OptionRule.TABLE).build();
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java
new file mode 100644
index 000000000..844a47563
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.db2.jdbc;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.NonNull;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class Db2JdbcDataSourceChannel implements DataSourceChannel {
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return Db2DataSourceConfig.OPTION_RULE;
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return Db2DataSourceConfig.METADATA_RULE;
+ }
+
+ @Override
+ public List getTables(
+ @NonNull String pluginName,
+ Map requestParams,
+ String database,
+ Map options) {
+ List tableNames = new ArrayList<>();
+ String filterName = options.get("filterName");
+ String size = options.get("size");
+ boolean isSize = StringUtils.isNotEmpty(size);
+ if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) {
+ filterName = "%" + filterName + "%";
+ } else if (StringUtils.equals(filterName, "")) {
+ filterName = null;
+ }
+ try (Connection connection = getConnection(requestParams);
+ ResultSet resultSet =
+ connection
+ .getMetaData()
+ .getTables(null, null, "%", new String[] {"TABLE"})) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("TABLE_NAME");
+ if (StringUtils.isNotBlank(tableName)) {
+ tableNames.add(tableName);
+ if (isSize && tableNames.size() >= Integer.parseInt(size)) {
+ break;
+ }
+ }
+ }
+ return tableNames;
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new DataSourcePluginException("get table names failed", e);
+ }
+ }
+
+ @Override
+ public List getDatabases(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+ // Hardcoded list of example database names
+ List dbNames = Arrays.asList("default");
+ return dbNames;
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+ try (Connection ignored = getConnection(requestParams)) {
+ return true;
+ } catch (Exception e) {
+ throw new DataSourcePluginException("check jdbc connectivity failed", e);
+ }
+ }
+
+ @Override
+ public List getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ List tableFields = new ArrayList<>();
+ try (Connection connection = getConnection(requestParams)) {
+ DatabaseMetaData metaData = connection.getMetaData();
+
+ // Retrieve primary key information
+ String primaryKey = getPrimaryKey(metaData, database, table);
+
+ // Retrieve column information
+ try (ResultSet resultSet = metaData.getColumns(null, null, table, null)) {
+
+ while (resultSet.next()) {
+ TableField tableField = new TableField();
+ String columnName = resultSet.getString("COLUMN_NAME");
+
+ // Set primary key flag
+ tableField.setPrimaryKey(primaryKey != null && primaryKey.equals(columnName));
+
+ // Set other field attributes
+ tableField.setName(columnName);
+ tableField.setType(resultSet.getString("TYPE_NAME"));
+ tableField.setComment(resultSet.getString("REMARKS"));
+
+ // Set nullable flag
+ String isNullable = resultSet.getString("IS_NULLABLE");
+ tableField.setNullable("YES".equalsIgnoreCase(isNullable));
+
+ tableFields.add(tableField);
+ }
+ }
+ } catch (SQLException e) {
+ // Log the exception and rethrow as DataSourcePluginException
+ System.out.println("Error while retrieving table fields: " + e);
+ throw new DataSourcePluginException("Failed to get table fields", e);
+ } catch (ClassNotFoundException e) {
+ // Log the exception and rethrow as DataSourcePluginException
+ System.out.println("JDBC driver class not found" + e);
+ throw new DataSourcePluginException("JDBC driver class not found", e);
+ }
+ return tableFields;
+ }
+
+ @Override
+ public Map> getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map requestParams,
+ @NonNull String database,
+ @NonNull List tables) {
+ return tables.parallelStream()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ table ->
+ getTableFields(
+ pluginName, requestParams, database, table)));
+ }
+
+ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ throws SQLException {
+ ResultSet primaryKeysInfo = metaData.getPrimaryKeys(null, dbName, tableName);
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
+ return null;
+ }
+
+ private Connection getConnection(Map requestParams)
+ throws SQLException, ClassNotFoundException {
+ // Ensure the DB2 JDBC driver is loaded
+ Class.forName("com.ibm.db2.jcc.DB2Driver");
+ checkNotNull(requestParams.get(Db2OptionRule.URL.key()), "Jdbc url cannot be null");
+ String url = requestParams.get(Db2OptionRule.URL.key());
+ if (requestParams.containsKey(Db2OptionRule.USER.key())) {
+ String username = requestParams.get(Db2OptionRule.USER.key());
+ String password = requestParams.get(Db2OptionRule.PASSWORD.key());
+ return DriverManager.getConnection(url, username, password);
+ }
+ return DriverManager.getConnection(url);
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java
similarity index 63%
rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java
index ee6aa4ba0..afdd75c95 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java
@@ -15,42 +15,34 @@
* limitations under the License.
*/
-package org.apache.seatunnel.datasource.plugin.redshift.s3;
+package org.apache.seatunnel.datasource.plugin.db2.jdbc;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo;
-import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum;
import com.google.auto.service.AutoService;
import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
import java.util.Set;
+@Slf4j
@AutoService(DataSourceFactory.class)
-public class S3RedshiftDataSourceFactory implements DataSourceFactory {
+public class Db2JdbcDataSourceFactory implements DataSourceFactory {
+
@Override
public String factoryIdentifier() {
- return "S3-Redshift";
+ return Db2DataSourceConfig.PLUGIN_NAME;
}
@Override
public Set supportedDataSources() {
- DataSourcePluginInfo s3DatasourcePluginInfo =
- DataSourcePluginInfo.builder()
- .name("S3-Redshift")
- .type(DatasourcePluginTypeEnum.DATABASE.getCode())
- .version("1.0.0")
- .supportVirtualTables(false)
- .icon("S3-Redshift")
- .icon("S3-Redshift")
- .build();
-
- return Sets.newHashSet(s3DatasourcePluginInfo);
+ return Sets.newHashSet(Db2DataSourceConfig.DB2_DATASOURCE_PLUGIN_INFO);
}
@Override
public DataSourceChannel createChannel() {
- return new S3RedshiftDataSourceChannel();
+ return new Db2JdbcDataSourceChannel();
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java
new file mode 100644
index 000000000..e5e2b9b1b
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.datasource.plugin.db2.jdbc;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+public class Db2OptionRule {
+
+ public static final Option URL =
+ Options.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "jdbc url, eg:"
+ + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
+
+ public static final Option USER =
+ Options.key("user").stringType().noDefaultValue().withDescription("jdbc user");
+
+ public static final Option PASSWORD =
+ Options.key("password").stringType().noDefaultValue().withDescription("jdbc password");
+
+ public static final Option DATABASE =
+ Options.key("database").stringType().noDefaultValue().withDescription("jdbc database");
+
+ public static final Option TABLE =
+ Options.key("table").stringType().noDefaultValue().withDescription("jdbc table");
+
+ public static final Option DRIVER =
+ Options.key("driver")
+ .enumType(DriverType.class)
+ .defaultValue(DriverType.DB2)
+ .withDescription("driver");
+
+ public enum DriverType {
+ DB2("com.ibm.db2.jcc.DB2Driver"),
+ ;
+ private final String driverClassName;
+
+ DriverType(String driverClassName) {
+ this.driverClassName = driverClassName;
+ }
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ @Override
+ public String toString() {
+ return driverClassName;
+ }
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml
index 32dce470d..f24fdb7c3 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml
@@ -33,4 +33,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml
index 479fc5984..8a1e0b6a7 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml
@@ -60,4 +60,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java
index fe7842f6b..24e863b28 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java
@@ -36,6 +36,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -58,17 +59,28 @@ public List getTables(
@NonNull String pluginName,
Map requestParams,
String database,
- Map option) {
+ Map options) {
List tableNames = new ArrayList<>();
+ String filterName = options.get("filterName");
+ String size = options.get("size");
+ boolean isSize = StringUtils.isNotEmpty(size);
+ if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) {
+ filterName = "%" + filterName + "%";
+ } else if (StringUtils.equals(filterName, "")) {
+ filterName = null;
+ }
try (Connection connection = getConnection(requestParams);
ResultSet resultSet =
connection
.getMetaData()
- .getTables(database, null, null, new String[] {"TABLE"})) {
+ .getTables(database, null, filterName, new String[] {"TABLE"})) {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
if (StringUtils.isNotBlank(tableName)) {
tableNames.add(tableName);
+ if (isSize && tableNames.size() >= Integer.parseInt(size)) {
+ break;
+ }
}
}
return tableNames;
@@ -176,11 +188,15 @@ private Connection getConnection(Map requestParams, String datab
String url =
JdbcUtils.replaceDatabase(
requestParams.get(MysqlOptionRule.URL.key()), databaseName);
+
+ Properties info = new java.util.Properties();
+ info.put("autoDeserialize", "false");
+ info.put("allowLoadLocalInfile", "false");
+ info.put("allowLoadLocalInfileInPath", "");
if (requestParams.containsKey(MysqlOptionRule.USER.key())) {
- String username = requestParams.get(MysqlOptionRule.USER.key());
- String password = requestParams.get(MysqlOptionRule.PASSWORD.key());
- return DriverManager.getConnection(url, username, password);
+ info.put("user", requestParams.get(MysqlOptionRule.USER.key()));
+ info.put("password", requestParams.get(MysqlOptionRule.PASSWORD.key()));
}
- return DriverManager.getConnection(url);
+ return DriverManager.getConnection(url, info);
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml
index 9e108276c..287b8f75c 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml
@@ -59,4 +59,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java
index 6844cd3a4..775decb1c 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java
@@ -26,6 +26,7 @@
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -33,12 +34,15 @@
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
+@Slf4j
public class OracleDataSourceChannel implements DataSourceChannel {
@Override
@@ -56,18 +60,72 @@ public List getTables(
@NonNull String pluginName,
Map requestParams,
String database,
- Map option) {
+ Map options) {
+ StringBuilder sqlWhere = new StringBuilder();
+ final String sql =
+ "SELECT * FROM ( SELECT OWNER, TABLE_NAME FROM ALL_TABLES\n"
+ + "WHERE TABLE_NAME NOT LIKE 'MDRT_%'\n"
+ + " AND TABLE_NAME NOT LIKE 'MDRS_%'\n"
+ + " AND TABLE_NAME NOT LIKE 'MDXT_%'\n"
+ + " AND (TABLE_NAME NOT LIKE 'SYS_IOT_OVER_%' AND IOT_NAME IS NULL)"
+ + "AND OWNER NOT IN ('APPQOSSYS', 'AUDSYS', 'CTXSYS', 'DVSYS', 'DBSFWUSER', 'DBSNMP',\n"
+ + " 'GSMADMIN_INTERNAL', 'LBACSYS', 'MDSYS', 'OJVMSYS', 'OLAPSYS',\n"
+ + " 'ORDDATA', 'ORDSYS', 'OUTLN', 'SYS', 'SYSTEM', 'WMSYS',\n"
+ + " 'XDB', 'EXFSYS', 'SYSMAN')";
+ sqlWhere.append(sql);
+ String filterName = options.get("filterName");
+ if (StringUtils.isNotEmpty(filterName)) {
+ String[] split = filterName.split("\\.");
+ if (split.length == 2) {
+ sqlWhere.append(" AND (TABLE_NAME LIKE '")
+ .append(
+ split[1].contains("%")
+ ? split[1].toUpperCase(Locale.ROOT)
+ : "%" + split[1].toUpperCase(Locale.ROOT) + "%")
+ .append("'")
+ .append(" AND OWNER LIKE '")
+ .append(
+ split[0].contains("%")
+ ? split[0].toUpperCase(Locale.ROOT)
+ : "%" + split[0].toUpperCase(Locale.ROOT) + "%")
+ .append("')");
+ } else {
+ String filterNameRep =
+ filterName.contains("%")
+ ? filterName.toUpperCase(Locale.ROOT)
+ : "%" + filterName.toUpperCase(Locale.ROOT) + "%";
+ sqlWhere.append(" AND (TABLE_NAME LIKE '%")
+ .append(filterNameRep)
+ .append("%'")
+ .append(" OR OWNER LIKE '%")
+ .append(filterNameRep)
+ .append("%')");
+ }
+ }
+ sqlWhere.append(" ORDER BY OWNER, TABLE_NAME ) ");
+ String size = options.get("size");
+ if (StringUtils.isNotEmpty(size)) {
+ sqlWhere.append("WHERE ROWNUM <= ").append(size);
+ }
+ log.info("execute sql :{}", sqlWhere.toString());
List tableNames = new ArrayList<>();
- try (Connection connection = getConnection(requestParams);
- ResultSet resultSet =
- connection
- .getMetaData()
- .getTables(database, null, null, new String[] {"TABLE"}); ) {
- while (resultSet.next()) {
- String tableName = resultSet.getString("TABLE_NAME");
- if (StringUtils.isNotBlank(tableName)) {
- tableNames.add(tableName);
+ long start = System.currentTimeMillis();
+ try (Connection connection = getConnection(requestParams); ) {
+ long end = System.currentTimeMillis();
+ log.info("connection, cost {}ms for oracle", end - start);
+ start = System.currentTimeMillis();
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sqlWhere.toString())) {
+ end = System.currentTimeMillis();
+ log.info("statement execute sql, cost {}ms for oracle", end - start);
+ start = System.currentTimeMillis();
+ while (resultSet.next()) {
+ String schemaName = resultSet.getString("OWNER");
+ String tableName = resultSet.getString("TABLE_NAME");
+ tableNames.add(schemaName + "." + tableName);
}
+ end = System.currentTimeMillis();
+ log.info("while result set, cost {}ms for oracle", end - start);
}
return tableNames;
} catch (ClassNotFoundException | SQLException e) {
@@ -80,13 +138,12 @@ public List getDatabases(
@NonNull String pluginName, @NonNull Map requestParams) {
List dbNames = new ArrayList<>();
try (Connection connection = getConnection(requestParams);
- PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;");
+ PreparedStatement statement =
+ connection.prepareStatement("SELECT NAME FROM v$database");
ResultSet re = statement.executeQuery()) {
- // filter system databases
while (re.next()) {
- String dbName = re.getString("database");
- if (StringUtils.isNotBlank(dbName)
- && !OracleDataSourceConfig.ORACLE_SYSTEM_DATABASES.contains(dbName)) {
+ String dbName = re.getString("NAME");
+ if (StringUtils.isNotBlank(dbName)) {
dbNames.add(dbName);
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml
index 9b07fb21a..27471250a 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml
@@ -58,4 +58,23 @@
provided
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java
index 9b2e9abfa..b21bf9930 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java
@@ -26,6 +26,7 @@
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -40,6 +41,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
+@Slf4j
public class PostgresqlDataSourceChannel implements DataSourceChannel {
@Override
@@ -57,18 +59,52 @@ public List getTables(
@NonNull String pluginName,
Map requestParams,
String database,
- Map option) {
+ Map options) {
List tableNames = new ArrayList<>();
- String query = "SELECT table_schema, table_name FROM information_schema.tables";
- try (Connection connection = getConnection(requestParams, database)) {
+ StringBuilder queryWhere = new StringBuilder();
+ String query =
+ "SELECT table_schema, table_name FROM information_schema.tables\n"
+ + "WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'root', 'pg_toast', 'pg_temp_1', 'pg_toast_temp_1', 'postgres', 'template0', 'template1')\n";
+ queryWhere.append(query);
+ String filterName = options.get("filterName");
+ if (StringUtils.isNotEmpty(filterName)) {
+ String[] split = filterName.split("\\.");
+ if (split.length == 2) {
+ queryWhere
+ .append("AND (table_schema LIKE '")
+ .append(split[0].contains("%") ? split[0] : "%" + split[0] + "%")
+ .append("'")
+ .append(" AND table_name LIKE '")
+ .append(split[1].contains("%") ? split[1] : "%" + split[1] + "%")
+ .append("')");
+ } else {
+ String filterNameRep =
+ filterName.contains("%") ? filterName : "%" + filterName + "%";
+ queryWhere
+ .append(" AND (table_schema LIKE '")
+ .append(filterNameRep)
+ .append("'")
+ .append(" OR table_name LIKE '")
+ .append(filterNameRep)
+ .append("')");
+ }
+ }
+ String size = options.get("size");
+ if (StringUtils.isNotEmpty(size)) {
+ queryWhere.append(" LIMIT ").append(size);
+ }
+ log.info(queryWhere.toString());
+ requestParams.put(
+ PostgresqlOptionRule.URL.key(),
+ JdbcUtils.replaceDatabase(
+ requestParams.get(PostgresqlOptionRule.URL.key()), database));
+ try (Connection connection = getConnection(requestParams)) {
try (Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(query)) {
+ ResultSet resultSet = statement.executeQuery(queryWhere.toString())) {
while (resultSet.next()) {
String schemaName = resultSet.getString("table_schema");
String tableName = resultSet.getString("table_name");
- if (StringUtils.isNotBlank(schemaName)
- && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains(
- schemaName)) {
+ if (StringUtils.isNotBlank(schemaName)) {
tableNames.add(schemaName + "." + tableName);
}
}
@@ -118,7 +154,11 @@ public List getTableFields(
@NonNull String database,
@NonNull String table) {
List tableFields = new ArrayList<>();
- try (Connection connection = getConnection(requestParams, database); ) {
+ requestParams.put(
+ PostgresqlOptionRule.URL.key(),
+ JdbcUtils.replaceDatabase(
+ requestParams.get(PostgresqlOptionRule.URL.key()), database));
+ try (Connection connection = getConnection(requestParams)) {
DatabaseMetaData metaData = connection.getMetaData();
String primaryKey = getPrimaryKey(metaData, database, table);
String[] split = table.split("\\.");
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml
index 440e4a451..7f48dca45 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml
@@ -59,4 +59,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml
index 1ff5b6bca..6ce0dd22c 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml
@@ -59,4 +59,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java
index cd3900912..e560e7f1f 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java
@@ -24,6 +24,7 @@
import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@@ -34,6 +35,7 @@
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -57,18 +59,66 @@ public List getTables(
@NonNull String pluginName,
Map requestParams,
String database,
- Map option) {
+ Map options) {
List tableNames = new ArrayList<>();
- try (Connection connection = getConnection(requestParams);
- ResultSet resultSet =
- connection
- .getMetaData()
- .getTables(database, null, null, new String[] {"TABLE"})) {
- while (resultSet.next()) {
- String tableName = resultSet.getString("TABLE_NAME");
- if (StringUtils.isNotBlank(tableName)) {
- tableNames.add(tableName);
+ StringBuilder queryWhere = new StringBuilder();
+ String query =
+ String.format(
+ "SELECT SCHEMA_NAME(schema_id) AS schema_name, name AS table_name\n"
+ + "FROM %s.sys.tables \n"
+ + "WHERE type = 'U' AND is_ms_shipped = 0 \n",
+ database);
+ queryWhere.append(query);
+ String filterName = options.get("filterName");
+ if (StringUtils.isNotEmpty(filterName)) {
+ String[] split = filterName.split("\\.");
+ if (split.length == 2) {
+ String formatStr =
+ " AND (name LIKE '"
+ + (split[1].contains("%") ? split[1] : "%" + split[1] + "%")
+ + "' AND SCHEMA_NAME(schema_id) LIKE '"
+ + (split[0].contains("%") ? split[0] : "%" + split[0] + "%")
+ + "')";
+ queryWhere.append(formatStr);
+ } else {
+ String filterNameRep =
+ filterName.contains("%") ? filterName : "%" + filterName + "%";
+ String formatStr =
+ " AND (name LIKE '"
+ + filterNameRep
+ + "' OR SCHEMA_NAME(schema_id) LIKE '"
+ + filterNameRep
+ + "')";
+ queryWhere.append(formatStr);
+ }
+ }
+ queryWhere.append("ORDER BY schema_name, table_name");
+ String size = options.get("size");
+ if (StringUtils.isNotEmpty(size)) {
+ queryWhere.append(" OFFSET 0 ROWS FETCH NEXT ").append(size).append(" ROWS ONLY");
+ }
+ log.info("execute sql :{}", queryWhere.toString());
+ long start = System.currentTimeMillis();
+ try (Connection connection = getConnection(requestParams)) {
+ long end = System.currentTimeMillis();
+ log.info("connection, cost {}ms for sqlserver", end - start);
+ start = System.currentTimeMillis();
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(queryWhere.toString())) {
+ end = System.currentTimeMillis();
+ log.info("execute sql, cost {}ms for sqlserver", end - start);
+ start = System.currentTimeMillis();
+ while (resultSet.next()) {
+ String schemaName = resultSet.getString("SCHEMA_NAME");
+ String tableName = resultSet.getString("table_name");
+ if (StringUtils.isNotBlank(schemaName)
+ && !SqlServerDataSourceConfig.SQLSERVER_SYSTEM_DATABASES.contains(
+ schemaName)) {
+ tableNames.add(schemaName + "." + tableName);
+ }
}
+ end = System.currentTimeMillis();
+ log.info("while result set, cost {}ms for sqlserver", end - start);
}
return tableNames;
} catch (ClassNotFoundException | SQLException e) {
@@ -115,11 +165,17 @@ public List getTableFields(
@NonNull Map requestParams,
@NonNull String database,
@NonNull String table) {
+ Pair pair = parseSchemaAndTable(table);
+ return getTableFields(requestParams, database, pair.getLeft(), pair.getRight());
+ }
+
+ private List getTableFields(
+ Map requestParams, String dbName, String schemaName, String tableName) {
List tableFields = new ArrayList<>();
- try (Connection connection = getConnection(requestParams, null)) {
+ try (Connection connection = getConnection(requestParams); ) {
DatabaseMetaData metaData = connection.getMetaData();
- String primaryKey = getPrimaryKey(metaData, database, table);
- try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) {
+ String primaryKey = getPrimaryKey(metaData, dbName, schemaName, tableName);
+ try (ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null)) {
while (resultSet.next()) {
TableField tableField = new TableField();
String columnName = resultSet.getString("COLUMN_NAME");
@@ -128,19 +184,33 @@ public List getTableFields(
tableField.setPrimaryKey(true);
}
tableField.setName(columnName);
- tableField.setType(resultSet.getString("TYPE_NAME"));
+ String typeString = resultSet.getString("TYPE_NAME");
+ String[] parts = typeString.split(" ");
+ String baseType = parts.length > 0 ? parts[0] : "";
+ tableField.setType(baseType);
tableField.setComment(resultSet.getString("REMARKS"));
Object nullable = resultSet.getObject("IS_NULLABLE");
- tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString()));
+ boolean isNullable = convertToBoolean(nullable);
+ tableField.setNullable(isNullable);
tableFields.add(tableField);
}
}
- } catch (ClassNotFoundException | SQLException e) {
+ } catch (Exception e) {
throw new DataSourcePluginException("get table fields failed", e);
}
return tableFields;
}
+ private boolean convertToBoolean(Object value) {
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ if (value instanceof String) {
+ return value.equals("TRUE");
+ }
+ return false;
+ }
+
@Override
public Map> getTableFields(
@NonNull String pluginName,
@@ -150,11 +220,13 @@ public Map> getTableFields(
return null;
}
- private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
+ private String getPrimaryKey(
+ DatabaseMetaData metaData, String dbName, String schemaName, String tableName)
throws SQLException {
- ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
- while (primaryKeysInfo.next()) {
- return primaryKeysInfo.getString("COLUMN_NAME");
+ try (ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName)) {
+ while (primaryKeysInfo.next()) {
+ return primaryKeysInfo.getString("COLUMN_NAME");
+ }
}
return null;
}
@@ -178,4 +250,12 @@ private Connection getConnection(Map requestParams, String datab
}
return DriverManager.getConnection(url);
}
+
+ private Pair parseSchemaAndTable(String tableName) {
+ String[] schemaAndTable = tableName.split("\\.");
+ if (schemaAndTable.length != 2) {
+ throw new DataSourcePluginException("table name is invalid");
+ }
+ return Pair.of(schemaAndTable[0], schemaAndTable[1]);
+ }
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java
index 271653388..b8b8f5e5a 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java
@@ -36,6 +36,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -169,11 +170,15 @@ private Connection getConnection(Map requestParams, String datab
String url =
JdbcUtils.replaceDatabase(
requestParams.get(StarRocksOptionRule.URL.key()), databaseName);
+
+ Properties info = new java.util.Properties();
+ info.put("autoDeserialize", "false");
+ info.put("allowLoadLocalInfile", "false");
+ info.put("allowLoadLocalInfileInPath", "");
if (requestParams.containsKey(StarRocksOptionRule.USER.key())) {
- String username = requestParams.get(StarRocksOptionRule.USER.key());
- String password = requestParams.get(StarRocksOptionRule.PASSWORD.key());
- return DriverManager.getConnection(url, username, password);
+ info.put("user", requestParams.get(StarRocksOptionRule.USER.key()));
+ info.put("password", requestParams.get(StarRocksOptionRule.PASSWORD.key()));
}
- return DriverManager.getConnection(url);
+ return DriverManager.getConnection(url, info);
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml
index 559a29361..477fad2ef 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml
@@ -58,4 +58,23 @@
provided
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java
index 90f929370..59a09ee97 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java
@@ -36,6 +36,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -176,11 +177,14 @@ private Connection getConnection(Map requestParams, String datab
String url =
JdbcUtils.replaceDatabase(
requestParams.get(TidbOptionRule.URL.key()), databaseName);
+ Properties info = new java.util.Properties();
+ info.put("autoDeserialize", "false");
+ info.put("allowLoadLocalInfile", "false");
+ info.put("allowLoadLocalInfileInPath", "");
if (requestParams.containsKey(TidbOptionRule.USER.key())) {
- String username = requestParams.get(TidbOptionRule.USER.key());
- String password = requestParams.get(TidbOptionRule.PASSWORD.key());
- return DriverManager.getConnection(url, username, password);
+ info.put("user", requestParams.get(TidbOptionRule.USER.key()));
+ info.put("password", requestParams.get(TidbOptionRule.PASSWORD.key()));
}
- return DriverManager.getConnection(url);
+ return DriverManager.getConnection(url, info);
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml
index ae68a64c5..8980db55d 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml
@@ -25,7 +25,7 @@
datasource-kafka
- 3.2.0
+ 3.4.0
@@ -61,4 +61,23 @@
${kafka.client.version}
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml
similarity index 58%
rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml
index 351f1cb6b..8bbf4dcf7 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml
@@ -22,7 +22,7 @@
${revision}
- datasource-s3redshift
+ datasource-mongodb
@@ -31,38 +31,34 @@
${project.version}
provided
+
- org.apache.seatunnel
- seatunnel-hadoop3-3.1.4-uber
- 2.3.2
-
-
- org.apache.avro
- avro
-
-
-
-
- com.amazon.redshift
- redshift-jdbc42
- ${redshift.version}
-
-
- org.apache.hadoop
- hadoop-aws
- ${hadoop-aws.version}
-
-
- jdk.tools
- jdk.tools
-
-
+ org.mongodb
+ mongodb-driver-sync
+ 4.7.1
+
- com.amazonaws
- aws-java-sdk-bundle
- ${aws-java-sdk-bundle.version}
+ org.apache.commons
+ commons-lang3
+
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java
new file mode 100644
index 000000000..55ba487d1
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.seatunnel.datasource.plugin.mongodb;
+
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
+import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
+import org.apache.seatunnel.datasource.plugin.api.model.TableField;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoIterable;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+@Slf4j
+public class MongoDataSoueceChannel implements DataSourceChannel {
+
+ private static final String DATABASE = "default";
+
+ @Override
+ public OptionRule getDataSourceOptions(@NonNull String pluginName) {
+ return MongoOptionRule.optionRule();
+ }
+
+ @Override
+ public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
+ return MongoOptionRule.metadataRule();
+ }
+
+ public List getTables(
+ @NonNull String pluginName,
+ Map requestParams,
+ String database,
+ Map options) {
+ checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default");
+
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List getDatabases(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+ return ImmutableList.of(DATABASE);
+ }
+
+ @Override
+ public List getTableFields(
+ @NonNull String pluginName,
+ @NonNull Map requestParams,
+ @NonNull String database,
+ @NonNull String table) {
+ checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean checkDataSourceConnectivity(
+ @NonNull String pluginName, @NonNull Map requestParams) {
+
+ try (MongoClient mongoClient = createMongoClient(requestParams)) {
+ // Verify if the connection to mongodb was successful
+ MongoIterable databaseNames = mongoClient.listDatabaseNames();
+ if (databaseNames.iterator().hasNext()) {
+ log.info("mongoDB connection successful");
+ return true;
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ throw new DataSourcePluginException(
+ "check MongoDB connectivity failed, " + e.getMessage(), e);
+ }
+ }
+
+ // Resolve the URI in requestParams of Map type
+ private MongoClient createMongoClient(Map requestParams) {
+
+ return MongoClients.create(
+ MongoRequestParamsUtils.parseStringFromRequestParams(requestParams));
+ }
+}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java
similarity index 66%
rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java
index ee6aa4ba0..b08f5b305 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.seatunnel.datasource.plugin.redshift.s3;
+package com.apache.seatunnel.datasource.plugin.mongodb;
import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory;
@@ -28,29 +28,30 @@
import java.util.Set;
@AutoService(DataSourceFactory.class)
-public class S3RedshiftDataSourceFactory implements DataSourceFactory {
+public class MongoDataSourceFactory implements DataSourceFactory {
+ public static final String MONGO_PLUGIN_NAME = "MongoDB";
+ public static final String MONGO_PLUGIN_ICON = "MongoDB";
+ public static final String MONGO_PLUGIN_VERSION = "1.0.0";
+
@Override
public String factoryIdentifier() {
- return "S3-Redshift";
+ return MONGO_PLUGIN_NAME;
}
@Override
public Set supportedDataSources() {
- DataSourcePluginInfo s3DatasourcePluginInfo =
+ return Sets.newHashSet(
DataSourcePluginInfo.builder()
- .name("S3-Redshift")
- .type(DatasourcePluginTypeEnum.DATABASE.getCode())
- .version("1.0.0")
- .supportVirtualTables(false)
- .icon("S3-Redshift")
- .icon("S3-Redshift")
- .build();
-
- return Sets.newHashSet(s3DatasourcePluginInfo);
+ .name(MONGO_PLUGIN_NAME)
+ .icon(MONGO_PLUGIN_ICON)
+ .version(MONGO_PLUGIN_VERSION)
+ .supportVirtualTables(true)
+ .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode())
+ .build());
}
@Override
public DataSourceChannel createChannel() {
- return new S3RedshiftDataSourceChannel();
+ return new MongoDataSoueceChannel();
}
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java
new file mode 100644
index 000000000..cde0f89fd
--- /dev/null
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.seatunnel.datasource.plugin.mongodb;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+
+public class MongoOptionRule {
+
+ public static final Option URI =
+ Options.key("uri")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The MongoDB connection uri.");
+
+ public static final Option DATABASE =
+ Options.key("database")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The name of MongoDB database to read or write.");
+
+ public static final Option COLLECTION =
+ Options.key("collection")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The name of MongoDB collection to read or write.");
+
+ public static OptionRule optionRule() {
+ return OptionRule.builder().required(URI).build();
+ }
+
+ public static OptionRule metadataRule() {
+ return OptionRule.builder().required(DATABASE, COLLECTION).build();
+ }
+}
diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java
similarity index 62%
rename from seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java
rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java
index 1dc971611..eb342acff 100644
--- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java
@@ -15,20 +15,19 @@
* limitations under the License.
*/
-package org.apache.seatunnel.scheduler.dolphinscheduler.dto;
-
-import lombok.Builder;
-import lombok.Data;
+package com.apache.seatunnel.datasource.plugin.mongodb;
import java.util.Map;
-@Data
-@Builder
-public class TaskDescriptionDto {
- private String name;
- private String executeScript;
- private String content;
- private Map params;
- private int retryInterval;
- private int retryTimes;
+import static com.google.common.base.Preconditions.checkArgument;
+
+public class MongoRequestParamsUtils {
+
+ public static String parseStringFromRequestParams(Map requestParams) {
+ checkArgument(
+ requestParams.containsKey(MongoOptionRule.URI.key()),
+ String.format("Missing %s in requestParams", MongoOptionRule.URI.key()));
+
+ return requestParams.get(MongoOptionRule.URI.key());
+ }
}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml
index 23fedfd59..885b5bdf7 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml
@@ -63,4 +63,22 @@
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ ${e2e.dependency.skip}
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java
index 4e5164fcf..e4a00fbdb 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java
@@ -38,6 +38,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
public class MysqlCDCDataSourceChannel implements DataSourceChannel {
@@ -65,8 +66,8 @@ public List getTables(
String pluginName,
Map requestParams,
String database,
- Map option) {
- return this.getTableNames(requestParams, database);
+ Map options) {
+ return this.getTableNames(requestParams, database, options);
}
@Override
@@ -154,13 +155,17 @@ protected Connection init(Map requestParams) throws SQLException
throw new DataSourcePluginException("Jdbc url is null");
}
String url = requestParams.get(MysqlCDCOptionRule.BASE_URL.key());
+
+ Properties info = new java.util.Properties();
+ info.put("autoDeserialize", "false");
+ info.put("allowLoadLocalInfile", "false");
+ info.put("allowLoadLocalInfileInPath", "");
if (null != requestParams.get(MysqlCDCOptionRule.PASSWORD.key())
&& null != requestParams.get(MysqlCDCOptionRule.USERNAME.key())) {
- String username = requestParams.get(MysqlCDCOptionRule.USERNAME.key());
- String password = requestParams.get(MysqlCDCOptionRule.PASSWORD.key());
- return DriverManager.getConnection(url, username, password);
+ info.put("user", requestParams.get(MysqlCDCOptionRule.USERNAME.key()));
+ info.put("password", requestParams.get(MysqlCDCOptionRule.PASSWORD.key()));
}
- return DriverManager.getConnection(url);
+ return DriverManager.getConnection(url, info);
}
protected List getDataBaseNames(Map requestParams) throws SQLException {
@@ -179,17 +184,29 @@ protected List getDataBaseNames(Map requestParams) throw
}
}
- protected List getTableNames(Map requestParams, String dbName) {
+ protected List getTableNames(
+ Map requestParams, String dbName, Map options) {
List tableNames = new ArrayList<>();
+ String filterName = options.get("filterName");
+ String size = options.get("size");
+ boolean isSize = StringUtils.isNotEmpty(size);
+ if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) {
+ filterName = "%" + filterName + "%";
+ } else if (StringUtils.equals(filterName, "")) {
+ filterName = null;
+ }
try (Connection connection = init(requestParams);
ResultSet resultSet =
connection
.getMetaData()
- .getTables(dbName, null, null, new String[] {"TABLE"})) {
+ .getTables(dbName, null, filterName, new String[] {"TABLE"})) {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
if (StringUtils.isNotBlank(tableName)) {
tableNames.add(tableName);
+ if (isSize && tableNames.size() >= Integer.parseInt(size)) {
+ break;
+ }
}
}
return tableNames;
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java
index 72a2ea208..8fb645d4a 100644
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java
+++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java
@@ -28,6 +28,8 @@
import java.sql.Connection;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
public interface DataSourceChannel {
@@ -71,11 +73,19 @@ List getTableFields(
@NonNull String database,
@NonNull String table);
- Map> getTableFields(
+ default Map> getTableFields(
@NonNull String pluginName,
@NonNull Map requestParams,
@NonNull String database,
- @NonNull List tables);
+ @NonNull List tables) {
+ return tables.parallelStream()
+ .collect(
+ Collectors.toMap(
+ Function.identity(),
+ table ->
+ getTableFields(
+ pluginName, requestParams, database, table)));
+ }
/**
* just check metadata field is right and used by virtual table
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java
deleted file mode 100644
index 8da509c12..000000000
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.datasource.plugin.redshift.s3;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.Arrays;
-import java.util.Map;
-
-import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
-
-@Slf4j
-public class HadoopS3AConfiguration {
-
- /* S3 constants */
- private static final String S3A_SCHEMA = "s3a";
- private static final String HDFS_S3N_IMPL = "org.apache.hadoop.fs.s3native.NativeS3FileSystem";
- private static final String HDFS_S3A_IMPL = "org.apache.hadoop.fs.s3a.S3AFileSystem";
- private static final String S3A_PROTOCOL = "s3a";
- private static final String DEFAULT_PROTOCOL = "s3n";
- private static final String S3_FORMAT_KEY = "fs.%s.%s";
- private static final String HDFS_IMPL_KEY = "impl";
-
- public static Configuration getConfiguration(Map s3Options) {
-
- if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) {
- throw new IllegalArgumentException(
- "S3Redshift datasource bucket is null, please check your config");
- }
- if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) {
- throw new IllegalArgumentException(
- "S3Redshift datasource endpoint is null, please check your config");
- }
- String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key());
-
- String protocol = DEFAULT_PROTOCOL;
- if (bucket.startsWith(S3A_PROTOCOL)) {
- protocol = S3A_PROTOCOL;
- }
- String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL;
- Configuration hadoopConf = new Configuration();
- hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket);
- hadoopConf.set(
- S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(),
- s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key()));
- hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl);
- if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) {
- Arrays.stream(
- s3Options
- .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())
- .split("\n"))
- .map(String::trim)
- .filter(StringUtils::isNotBlank)
- .forEach(
- line -> {
- String[] kv = line.split("=");
- if (kv.length == 2) {
- hadoopConf.set(kv[0].trim(), kv[1].trim());
- }
- });
- }
- if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider
- .getProvider()
- .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) {
- hadoopConf.set(
- S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
- s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
- hadoopConf.set(
- "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key()));
- hadoopConf.set(
- "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key()));
- } else {
- hadoopConf.set(
- S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
- s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
- }
- return hadoopConf;
- }
-
- private static String formatKey(String protocol, String key) {
- return String.format(S3_FORMAT_KEY, protocol, key);
- }
-}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java
deleted file mode 100644
index de9e2b09e..000000000
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.datasource.plugin.redshift.s3;
-
-import org.apache.seatunnel.api.configuration.util.OptionRule;
-import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel;
-import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException;
-import org.apache.seatunnel.datasource.plugin.api.model.TableField;
-import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-import com.google.common.collect.Sets;
-import lombok.NonNull;
-import lombok.extern.slf4j.Slf4j;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-@Slf4j
-public class S3RedshiftDataSourceChannel implements DataSourceChannel {
-
- @Override
- public OptionRule getDataSourceOptions(@NonNull String pluginName) {
- return S3RedshiftOptionRule.optionRule();
- }
-
- @Override
- public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) {
- return S3RedshiftOptionRule.metadataRule();
- }
-
- @Override
- public List getTables(
- @NonNull String pluginName, Map requestParams, String database) {
- return getTableNames(requestParams, database);
- }
-
- @Override
- public List getDatabases(
- @NonNull String pluginName, @NonNull Map requestParams) {
- try {
- return getDataBaseNames(pluginName, requestParams);
- } catch (SQLException e) {
- throw new DataSourcePluginException("Query redshift databases failed", e);
- }
- }
-
- @Override
- public boolean checkDataSourceConnectivity(
- @NonNull String pluginName, @NonNull Map requestParams) {
- checkHdfsS3Connection(requestParams);
- checkJdbcConnection(requestParams);
- return true;
- }
-
- @Override
- public List getTableFields(
- @NonNull String pluginName,
- @NonNull Map requestParams,
- @NonNull String database,
- @NonNull String table) {
- return getTableFields(requestParams, database, table);
- }
-
- @Override
- public Map> getTableFields(
- @NonNull String pluginName,
- @NonNull Map requestParams,
- @NonNull String database,
- @NonNull List tables) {
- // not need this method
- return null;
- }
-
- private void checkJdbcConnection(Map requestParams) {
- String jdbcUrl = requestParams.get(S3RedshiftOptionRule.JDBC_URL.key());
- String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key());
- String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key());
- if (StringUtils.isBlank(jdbcUrl)) {
- throw new DataSourcePluginException("Redshift Jdbc url is empty");
- }
- if (StringUtils.isBlank(username) && StringUtils.isBlank(password)) {
- try (Connection ignored = DriverManager.getConnection(jdbcUrl)) {
- log.info("Redshift jdbc connection is valid");
- return;
- } catch (SQLException e) {
- throw new DataSourcePluginException(
- "Check Redshift jdbc connection failed,please check your config", e);
- }
- }
- try (Connection ignored = DriverManager.getConnection(jdbcUrl, username, password)) {
- log.info("Redshift jdbc connection is valid");
- } catch (SQLException e) {
- throw new DataSourcePluginException(
- "Check Redshift jdbc connection failed,please check your config", e);
- }
- }
-
- private void checkHdfsS3Connection(Map requestParams) {
- Configuration s3Conf = HadoopS3AConfiguration.getConfiguration(requestParams);
- try (FileSystem fs = FileSystem.get(s3Conf)) {
- fs.getFileStatus(new org.apache.hadoop.fs.Path("/"));
- } catch (IOException e) {
- throw new DataSourcePluginException(
- "S3 configuration is invalid, please check your config", e);
- }
- }
-
- protected Connection init(Map requestParams, String databaseName)
- throws SQLException {
- if (null == requestParams.get(S3RedshiftOptionRule.JDBC_URL.key())) {
- throw new DataSourcePluginException("Jdbc url is null");
- }
- String url =
- JdbcUtils.replaceDatabase(
- requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName);
- if (null != requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key())
- && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) {
- String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key());
- String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key());
- return DriverManager.getConnection(url, username, password);
- }
- return DriverManager.getConnection(url);
- }
-
- protected List getDataBaseNames(String pluginName, Map requestParams)
- throws SQLException {
- List dbNames = new ArrayList<>();
- try (Connection connection = init(requestParams, null);
- PreparedStatement statement =
- connection.prepareStatement("select datname from pg_database;");
- ResultSet re = statement.executeQuery()) {
- while (re.next()) {
- String dbName = re.getString("datname");
- if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) {
- dbNames.add(dbName);
- }
- }
- return dbNames;
- } catch (SQLException e) {
- throw new DataSourcePluginException("get databases failed", e);
- }
- }
-
- protected List getTableNames(Map requestParams, String dbName) {
- List tableNames = new ArrayList<>();
- try (Connection connection = init(requestParams, dbName); ) {
- ResultSet resultSet =
- connection.getMetaData().getTables(dbName, null, null, new String[] {"TABLE"});
- while (resultSet.next()) {
- String tableName = resultSet.getString("TABLE_NAME");
- if (StringUtils.isNotBlank(tableName)) {
- tableNames.add(tableName);
- }
- }
- return tableNames;
- } catch (SQLException e) {
- throw new DataSourcePluginException("get table names failed", e);
- }
- }
-
- protected List getTableFields(
- Map requestParams, String dbName, String tableName) {
- List tableFields = new ArrayList<>();
- try (Connection connection = init(requestParams, dbName); ) {
- DatabaseMetaData metaData = connection.getMetaData();
- String primaryKey = getPrimaryKey(metaData, dbName, tableName);
- String[] split = tableName.split("\\.");
- if (split.length != 2) {
- throw new DataSourcePluginException(
- "Postgresql tableName should composed by schemaName.tableName");
- }
- ResultSet resultSet = metaData.getColumns(dbName, split[0], split[1], null);
- while (resultSet.next()) {
- TableField tableField = new TableField();
- String columnName = resultSet.getString("COLUMN_NAME");
- tableField.setPrimaryKey(false);
- if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) {
- tableField.setPrimaryKey(true);
- }
- tableField.setName(columnName);
- tableField.setType(resultSet.getString("TYPE_NAME"));
- tableField.setComment(resultSet.getString("REMARKS"));
- Object nullable = resultSet.getObject("IS_NULLABLE");
- boolean isNullable = convertToBoolean(nullable);
- tableField.setNullable(isNullable);
- tableFields.add(tableField);
- }
- } catch (SQLException e) {
- throw new DataSourcePluginException("get table fields failed", e);
- }
- return tableFields;
- }
-
- private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName)
- throws SQLException {
- ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName);
- while (primaryKeysInfo.next()) {
- return primaryKeysInfo.getString("COLUMN_NAME");
- }
- return null;
- }
-
- @SuppressWarnings("checkstyle:MagicNumber")
- private static boolean checkHostConnectable(String host, int port) {
- try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(host, port), 1000);
- return true;
- } catch (IOException e) {
-
- throw new DataSourcePluginException("check host connectable failed", e);
- }
- }
-
- private boolean isNotSystemDatabase(String dbName) {
- return !POSTGRESQL_SYSTEM_DATABASES.contains(dbName.toLowerCase());
- }
-
- private boolean convertToBoolean(Object value) {
- if (value instanceof Boolean) {
- return (Boolean) value;
- }
- if (value instanceof String) {
- return value.equals("TRUE");
- }
- return false;
- }
-
- public static final Set POSTGRESQL_SYSTEM_DATABASES =
- Sets.newHashSet(
- "information_schema",
- "pg_catalog",
- "root",
- "pg_toast",
- "pg_temp_1",
- "pg_toast_temp_1",
- "postgres",
- "template0",
- "template1");
-}
diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java
deleted file mode 100644
index 5d69ee803..000000000
--- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.seatunnel.datasource.plugin.redshift.s3;
-
-import org.apache.seatunnel.api.configuration.Option;
-import org.apache.seatunnel.api.configuration.Options;
-import org.apache.seatunnel.api.configuration.util.OptionRule;
-
-import java.util.Arrays;
-import java.util.Map;
-
-public class S3RedshiftOptionRule {
-
- public static final Option JDBC_URL =
- Options.key("jdbc_url")
- .stringType()
- .noDefaultValue()
- .withDescription(
- "Redshift jdbc connection url, eg: jdbc:redshift://localhost:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8");
-
- public static final Option JDBC_USER =
- Options.key("jdbc_user")
- .stringType()
- .noDefaultValue()
- .withDescription("Redshift jdbc connection user");
-
- public static final Option JDBC_PASSWORD =
- Options.key("jdbc_password")
- .stringType()
- .noDefaultValue()
- .withDescription("Redshift jdbc connection password");
-
- public static final Option ACCESS_KEY =
- Options.key("access_key")
- .stringType()
- .noDefaultValue()
- .withDescription("S3 access key");
-
- public static final Option SECRET_KEY =
- Options.key("secret_key")
- .stringType()
- .noDefaultValue()
- .withDescription("S3 secret key");
-
- public static final Option BUCKET =
- Options.key("bucket").stringType().noDefaultValue().withDescription("S3 bucket name");
-
- public static final Option FS_S3A_ENDPOINT =
- Options.key("fs.s3a.endpoint")
- .stringType()
- .noDefaultValue()
- .withDescription("fs s3a endpoint");
-
- public static final Option S3A_AWS_CREDENTIALS_PROVIDER =
- Options.key("fs.s3a.aws.credentials.provider")
- .enumType(S3aAwsCredentialsProvider.class)
- .defaultValue(S3aAwsCredentialsProvider.InstanceProfileCredentialsProvider)
- .withDescription("s3a aws credentials provider");
-
- public static final Option