diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 119698e97..39f5b8790 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -32,11 +32,11 @@ body: attributes: label: Search before asking description: > - Please make sure to search in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) + Please make sure to search in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) first to see whether the same issue was reported already. options: - label: > - I had searched in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found + I had searched in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found no similar issues. required: true diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 2a9877e3f..3d4350c4a 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -30,11 +30,11 @@ body: attributes: label: Search before asking description: > - Please make sure to search in the [feature](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) first + Please make sure to search in the [feature](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) first to see whether the same feature was requested already. options: - label: > - I had searched in the [feature](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) and found no + I had searched in the [feature](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22Feature%22) and found no similar feature requirement. required: true diff --git a/.github/ISSUE_TEMPLATE/umbrella.yml b/.github/ISSUE_TEMPLATE/umbrella.yml index 8db9b4dc1..af553ece2 100644 --- a/.github/ISSUE_TEMPLATE/umbrella.yml +++ b/.github/ISSUE_TEMPLATE/umbrella.yml @@ -34,11 +34,11 @@ body: attributes: label: Search before asking description: > - Please make sure to search in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) + Please make sure to search in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) first to see whether the same issue was reported already. options: - label: > - I had searched in the [issues](https://github.com/apache/incubator-seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found + I had searched in the [issues](https://github.com/apache/seatunnel/issues?q=is%3Aissue+label%3A%22bug%22) and found no similar issues. required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3752eb8b0..66f08d31c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,7 @@ Feel free to ping committers for the review! ## Contribution Checklist - - Make sure that the pull request corresponds to a [GITHUB issue](https://github.com/apache/incubator-seatunnel/issues). + - Make sure that the pull request corresponds to a [GITHUB issue](https://github.com/apache/seatunnel/issues). - Name the pull request in the form "[Feature] [component] Title of the pull request", where *Feature* can be replaced by `Hotfix`, `Bug`, etc. @@ -24,5 +24,5 @@ Feel free to ping committers for the review! * [ ] Code changed are covered with tests, or it does not need tests for reason: * [ ] If any new Jar binary package adding in your PR, please add License Notice according - [New License Guide](https://github.com/apache/incubator-seatunnel/blob/dev/docs/en/contribution/new-license.md) -* [ ] If necessary, please update the documentation to describe the new feature. https://github.com/apache/incubator-seatunnel/tree/dev/docs + [New License Guide](https://github.com/apache/seatunnel/blob/dev/docs/en/contribution/new-license.md) +* [ ] If necessary, please update the documentation to describe the new feature. https://github.com/apache/seatunnel/tree/dev/docs diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index d2ff663a1..05070c59b 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -131,7 +131,7 @@ jobs: cache: 'maven' - name: Install run: >- - ./mvnw -B -q install -DskipTests -P release + ./mvnw -B -q install -DskipTests -Prelease -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true diff --git a/README.md b/README.md index 73ab7e35d..0eae9fc6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ seatunnel logo -[![Backend Workflow](https://github.com/apache/incubator-seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/incubator-seatunnel/actions/workflows/backend.yml) +[![Backend Workflow](https://github.com/apache/seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/seatunnel/actions/workflows/backend.yml) [![Slack](https://img.shields.io/badge/slack-%23seatunnel-4f8eba?logo=slack)](https://join.slack.com/t/apacheseatunnel/shared_invite/zt-123jmewxe-RjB_DW3M3gV~xL91pZ0oVQ) [![Twitter Follow](https://img.shields.io/twitter/follow/ASFSeaTunnel.svg?label=Follow&logo=twitter)](https://twitter.com/ASFSeaTunnel) @@ -16,7 +16,7 @@ SeaTunnel was formerly named Waterdrop , and renamed SeaTunnel since October 12, So, What we are? An open-source web console to manage your seatunnel-script, and would push them to any scheduling-system easily. -Click it if your want to know more about our design. 👉🏻[Design](https://github.com/apache/incubator-seatunnel/issues/1947) +Click it if your want to know more about our design. 👉🏻[Design](https://github.com/apache/seatunnel/issues/1947) ## How to start @@ -25,28 +25,37 @@ Notice: Some details please refer to the docs/QuickStart.md ### 1 Preparing the Apache SeaTunnel environment -#### 1.1 Install Apache SeaTunnel +Because SeaTunnel Web uses the SeaTunnel Java Client to submit jobs, running SeaTunnel Web requires preparing a SeaTunnel Zeta Engine service first. -If you already have Apache SeaTunnel environment, you can skip this step and go to [Install your SeaTunnel to your local maven repository](#1.2 Install your SeaTunnel to your local maven repository) +Based on the usage requirements of SeaTunnel Zeta Engine, the SeaTunnel Client node that submits the job must have the same operating system and installation directory structure as the SeaTunnel Server node that runs the job. Therefore, if you want to run SeaTunnel Web in IDEA, you must install and run a SeaTunnel Zeta Engine Server on the same machine as the IDEA. -Because running SeaTunnel Web must rely on the SeaTunnel, if you do not have a SeaTunnel environment, you need to first install and deploy a SeaTunnel (hereinafter referred to as ST). You need to use the dev branch in the ST repositories. +Don't worry, the next steps will tell you how to correctly install SeaTunnel Zeta Engine Server in different situations. -Reference `https://github.com/apache/seatunnel` to deploy a ST - -#### 1.2 Install your SeaTunnel to your local maven repository +### 2 Run SeaTunnel Web in IDEA -If you already have a ST environment which is from ST github dev branch. Make sure the version is 2.3.2-SNAPSHOT in the pom file. then use command './mvnw clean install '-Dmaven.test.skip=true' '-Dcheckstyle.skip=true'' to install ST package into your local maven repository +If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server) +#### 2.1 Install SeaTunnel Zeta Engine Server +You have two ways to get the SeaTunnel installer package. Build from source code or download from the SeaTunnel website. -#### 1.3 Run SeaTunnel in RDEA +**The SeaTunnel version used here is only for writing this document to show you the process used, and does not necessarily represent the correct version. SeaTunnel Web and SeaTunnel Engine have strict version dependencies, and you can confirm the specific version mapping through xxx. Now only support build SeaTunnel Web and Seatunnel Zeta Engine in local, Because it is necessary to ensure that the seatunnel-api in SeaTunnel Web and the version in SeaTunnel Zeta Engine are the same.** -Reference `https://github.com/apache/seatunnel` +##### 2.1.1 Build from source code and deploy +* Get the source package from https://seatunnel.apache.org/download or https://github.com/apache/seatunnel.git +* Build installer package use maven command `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" ` +* After building, it is necessary to set an environment variable `ST_WEB_BASEDIR_PATH` to represent the location of the data source shade package. A custom class loader will be used to load the data source shade package based on this. For example: `ST_WEB_BASEDIR_PATH=/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/` +* Then you can get the installer package in `${Your_code_dir}/seatunnel-dist/target`, For example:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` +* Run `tar -zxvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` to unzip the installer package. +* Run `cd apache-seatunnel-2.3.3-SNAPSHOT & sh bin/seatunnel-cluster.sh -d` to run the SeaTunnel Zeta Engine Server. +* Please confirm that port 5801 is being monitored by the SeaTunnelServer process. -### 2 Run SeaTunnel Web in IDEA +##### 2.1.2 Download installer package and deploy +The other way to install SeaTunnel Zeta Engine Server is download the installer package from https://seatunnel.apache.org/download and deploy. -If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server) +* Download and install connector plugin(Some third-party dependency packages will also be automatically downloaded and installed during this process, such as hadoop jar). You can get the step from https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment. +* Run `cd apache-seatunnel-2.3.2 & sh bin/seatunnel-cluster.sh -d` to run the SeaTunnel Zeta Engine Server. -#### 2.1 Init database +#### 2.2 Init database 1. Edit `seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_env.sh` file, Complete the installed database address, port, username, and password. Here is an example: @@ -58,14 +67,27 @@ If you want to deploy and run SeaTunnel Web, Please turn to [3 Run SeaTunnel Web ``` 2. Run init shell `sh seatunnel-server/seatunnel-app/src/main/resources/script/init_sql.sh` If there are no errors during operation, it indicates successful initialization. -#### 2.2 Config application and Run SeaTunnel Web Backend Server +#### 2.3 Build the project + +```shell +sh build.sh code +``` + +#### 2.4 Config application and Run SeaTunnel Web Backend Server 1. Edit `seatunnel-server/seatunnel-app/src/main/resources/application.yml` Fill in the database connection information +2. Edit `seatunnel-server/seatunnel-app/src/main/resources/application.yml`, add `jwt.secretKey` value. Eg: SeaTunnel. ![img.png](docs/images/application_config.png) +2. Copy `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` file to `seatunnel-web/seatunnel-server/seatunnel-app/src/main/resources` dir. +3. Run `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` If there are no errors reported, the seatunnel web backend service is successfully started. Notice that, you must set `-DSEATUNNEL_HOME=${your_seatunnel_install_path}` like this: + +![img.png](docs/images/idea_st_home.png) -2. Run `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` If there are no errors reported, the seatunnel web backend service is successfully started. +Because the data source plugin is dynamically loaded, it is necessary to set relevant environment variables: + +![img.png](docs/images/st_web_basedir_path.png) #### 2.3 Run SeaTunnel Web Front End @@ -88,17 +110,46 @@ If there are no issues with the operation, the following information will be dis Accessing in a browser http://127.0.0.1:5173/login Okay, the default username and password are admin/admin. ### 3 Run SeaTunnel Web In Server +To run SeaTunnel Web on the server, you need to first have a SeaTunnel Zeta Engine Server environment. If you do not already have one, you can refer to the following steps for deployment. + +#### 3.1 Deploy SeaTunnel Zeta Engine Server In Server Node + +You have two ways to get the SeaTunnel installer package. Build from source code or download from the SeaTunnel website. + +**The SeaTunnel version used here is only for writing this document to show you the process used, and does not necessarily represent the correct version. SeaTunnel Web and SeaTunnel Engine have strict version dependencies, and you can confirm the specific version mapping through xxx** + +##### 3.1.1 Build from source code +* Get the source package from https://seatunnel.apache.org/download or https://github.com/apache/seatunnel.git +* Build installer package use maven command `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" ` +* Then you can get the installer package in `${Your_code_dir}/seatunnel-dist/target`, For example:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` + +##### 3.1.2 Download installer package +The other way to get SeaTunnel Zeta Engine Server installer package is download the installer package from https://seatunnel.apache.org/download and install plugins online. -#### 3.1 Build Install Package From Code +* Download and install connector plugin(Some third-party dependency packages will also be automatically downloaded and installed during this process, such as hadoop jar). You can get the step from https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment. +* After completing the previous step, you will receive an installation package that can be used to install SeaTunnel Zeta Engine Server on the server. Run `tar -zcvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz apache-seatunnel-2.3.3-SNAPSHOT` + +##### 3.1.3 Deploy SeaTunnel Zeta Server +After 3.1.1 or 3.1.2 you can get an installer package `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`, Then you can copy it to you server node and deploy reference https://seatunnel.apache.org/docs/seatunnel-engine/deployment. + +##### 3.1.4 Deploy SeaTunnel Zeta Client In SeaTunnel Web Run Node +If you use SeaTunnel Web, you need deploy a SeaTunnel Zeta Client in the SeaTunnel Web run Node. **If you run SeaTunnel Zeta Server and SeaTunnel Web in same node, you can skip this step**. + +* Copy `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` to the SeaTunnel Web node and unzip it **in the same path of SeaTunnel Zeta Server node**. +* Set `SEATUNNEL_HOME` to environment variable like SeaTunnel Zeta Server node. +* Config `hazelcast-client.yaml` reference https://seatunnel.apache.org/docs/seatunnel-engine/deployment#6-config-seatunnel-engine-client +* Run `$SEATUNNEL_HOME/bin/seatunnel.sh --config $SEATUNNEL_HOME/config/v2.batch.config.template`, If this job run finished, it indicates successful client deployment. + +#### 3.2 Build SeaTunnel Web Install Package From Code ``` -cd incubator-seatunnel-web +cd seatunnel-web sh build.sh code ``` -Then you can find the installer package in dir `incubator-seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz`. +Then you can find the installer package in dir `seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz`. -#### 3.2 Install +#### 3.3 Install Copy the `apache-seatunnel-web-${project.version}.tar.gz` to your server node and unzip it. @@ -106,7 +157,7 @@ Copy the `apache-seatunnel-web-${project.version}.tar.gz` to your server node an tar -zxvf apache-seatunnel-web-${project.version}.tar.gz ``` -#### 3.3 Init database +#### 3.4 Init database 1. Edit `apache-seatunnel-web-${project.version}/script/seatunnel_server_env.sh` file, Complete the installed database address, port, username, and password. Here is an example: @@ -118,13 +169,16 @@ tar -zxvf apache-seatunnel-web-${project.version}.tar.gz ``` 2. Run init shell `sh apache-seatunnel-web-${project.version}/script/init_sql.sh` If there are no errors during operation, it indicates successful initialization. -#### 3.4 Config application and Run SeaTunnel Web Backend Server +#### 3.5 Config application and Run SeaTunnel Web Backend Server -Edit `apache-seatunnel-web-${project.version}/config/application.yml` Fill in the database connection information and DS interface related information in the file. +* Edit `apache-seatunnel-web-${project.version}/conf/application.yml` Fill in the database connection information and DS interface related information in the file. ![image](docs/images/application_config.png) -#### 3.5 Start SeaTunnel Web +* Copy `$SEATUNNEL_HOME/config/hazelcast-client.yaml` to `apache-seatunnel-web-${project.version}/conf/` +* Copy `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` file to `apache-seatunnel-web-${project.version}/conf/` dir. + +#### 3.6 Start SeaTunnel Web ```shell cd apache-seatunnel-web-${project.version} diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 000000000..1e3bd9cfe --- /dev/null +++ b/README_CN.md @@ -0,0 +1,208 @@ +# Apache SeaTunnel + +seatunnel logo + +[![Backend Workflow](https://github.com/apache/seatunnel/actions/workflows/backend.yml/badge.svg?branch=dev)](https://github.com/apache/seatunnel/actions/workflows/backend.yml) +[![Slack](https://img.shields.io/badge/slack-%23seatunnel-4f8eba?logo=slack)](https://join.slack.com/t/apacheseatunnel/shared_invite/zt-123jmewxe-RjB_DW3M3gV~xL91pZ0oVQ) +[![Twitter Follow](https://img.shields.io/twitter/follow/ASFSeaTunnel.svg?label=Follow&logo=twitter)](https://twitter.com/ASFSeaTunnel) + +--- +[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) + +SeaTunnel的前身是Waterdrop，并于2021年10月12日更名为SeaTunnel。 + +--- + +那么，SeaTunnel是什么? +SeaTunnel是下一代超高性能、分布式、海量数据集成工具。它每天可以稳定高效地同步数百亿个数据，并已在许多公司的生产中使用。SeaTunnel-web为SeaTunnel提供可视化界面，方便用户使用Apache SeaTunnel。 + +如果你想知道更多关于SeaTunnel的设计。 👉🏻[设计](https://github.com/apache/seatunnel/issues/1947) + + +## 如何开始 + +注意:一些细节请参考 docs/QuickStart.md + +### 1 准备Apache SeaTunnel环境 + +因为SeaTunnel Web使用SeaTunnel Java客户端来提交作业，所以运行SeaTunnel Web需要首先准备SeaTunnel Zeta Engine服务。 + +根据SeaTunnel Zeta Engine的使用要求，提交作业的SeaTunnel Client节点必须与运行作业的SeaTunnel Server节点具有相同的操作系统和安装目录结构。比如如果你想在本地IDEA中运行SeaTunnel Web，你必须在与IDEA相同的机器上安装和运行SeaTunnel Zeta引擎服务器。 + +别担心，接下来的步骤将告诉您如何在不同情况下正确安装SeaTunnel Zeta Engine Server。 +### 2 在IDEA中运行SeaTunnel Web + +如果您想部署和运行SeaTunnel Web，请转到 [3 Run SeaTunnel Web In Server](#3 Run SeaTunnel Web In Server) + +#### 2.1 安装SeaTunnel Zeta引擎服务器 +有两种方法可以获得SeaTunnel安装包。从源代码构建或从SeaTunnel网站下载。 + +**此处使用的SeaTunnel版本仅用于编写本文档以向您展示所使用的过程，并不一定代表正确的版本。SeaTunnel Web和SeaTunnel Engine有严格的版本依赖关系，您可以通过确认具体的版本对应关系。现在只支持在本地构建SeaTunnel Web和SeaTunnel Zeta Engine，因为有必要确保SeaTunnel Web中的SeaTunnel-api与SeaTunnel Zeta Engine中的版本相同** +##### 2.1.1 从源代码构建和部署 +* 从https://seatunnel.apache.org/download 或 https://github.com/apache/seatunnel.git 获取源包 +* 构建安装包使用maven命令。`./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" ` +* 在构建之后，需要设置一个环境变量`ST_WEB_BASEDIR_PATH`来表示数据源shade包的位置。将使用自定义类加载器来基于此加载数据源shade包。例如:`ST_WEB_BASEDIR_PATH=/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/` +然后你可以在`${Your_code_dir}/seatunnel-dist/target`下获取安装包，例如:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`。 +* 执行`tar -zxvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`解压安装包。 +* 运行`cd apache-seatunnel-2.3.3-SNAPSHOT & sh bin/seatunnel-cluster.sh -d`运行SeaTunnel Zeta Engine Server。 +* SeaTunnel Zeta Engine Server默认占用5801端口，请确认端口5801正在被SeaTunnelServer进程占用。 + +##### 2.1.2 下载安装程序包并进行部署 +下载安装包并部署SeaTunnel Zeta Engine Server的另一种安装方式是从https://seatunnel.apache.org/download 下载安装包并部署。 + +* 下载并安装连接器插件(一些第三方依赖包也会在此过程中自动下载并安装，如Hadoop jar)。您可以从https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment 获得该步骤。 +* 运行`cd apache-seatunnel-2.3.2 & sh bin/seatunnel-cluster.sh -d`运行seattunnel Zeta Engine Server。 +#### 2.2 初始化数据库 + +1. 编辑 `seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_env.sh` 文件, 填写已安装的数据库 address, port, username, and password. 下面是一个例子: + + ``` + export HOSTNAME="localhost" + export PORT="3306" + export USERNAME="root" + export PASSWORD="123456" + ``` +2. 执行命令 `sh seatunnel-server/seatunnel-app/src/main/resources/script/init_sql.sh` 如果运行过程中没有错误，则说明初始化成功。 + +#### 2.3 构建项目 + +```shell +sh build.sh code +``` + +#### 2.4 配置应用程序并运行SeaTunnel Web后端服务器 + +1. 编辑 `seatunnel-server/seatunnel-app/src/main/resources/application.yml` 写数据库连接信息 + +![img.png](docs/images/application_config.png) + +2. 复制 `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` 文件 到 `seatunnel-web/seatunnel-server/seatunnel-app/src/main/resources` 目录. +3. 运行 `seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java` 如果没有报错，说明seatunnel web后端服务启动成功。注意，你必须设置 `-DSEATUNNEL_HOME=${your_seatunnel_install_path}` 像这样: + +![img.png](docs/images/idea_st_home.png) + +由于数据源插件是动态加载的，所以需要设置相关的环境变量: + +![img.png](docs/images/st_web_basedir_path.png) + +#### 2.3 运行SeaTunnel Web Front End + +``` +cd seatunnel-ui +npm install +npm run dev + +``` + +如果操作正常，系统显示如下信息: + +``` + ➜ Local: http://127.0.0.1:5173/ + ➜ Network: use --host to expose + ➜ press h to show help + +``` + +在浏览器中访问 `http://127.0.0.1:5173/login` , 默认用户名和密码是 `admin/admin` + +### 3 在服务器上运行 SeaTunnel Web +要在服务器上运行SeaTunnel Web，首先需要有一个SeaTunnel Zeta引擎服务器环境。如果还没有，可以参考以下步骤进行部署。 + +#### 3.1 在Server Node中部署SeaTunnel Zeta Engine Server + +有两种方法可以获得SeaTunnel安装包。从源代码构建或从SeaTunnel网站下载。 + +**此处使用的SeaTunnel版本仅用于编写本文档以向您展示所使用的过程，并不一定代表正确的版本。SeaTunnel Web和SeaTunnel Engine有严格的版本依赖关系，您可以通过xxx确认具体的版本对应关系** + +##### 3.1.1 从源码编译 +* 从 https://seatunnel.apache.org/download 或 https://github.com/apache/seatunnel.git 获取源码包 +* 使用maven命令编译安装包 `./mvnw -U -T 1C clean install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"checkstyle.skip"=true -D"license.skipAddThirdParty" ` +* 然后您可以在`${您的代码目录}/seatunnel-dist/target`中获得安装包,例如:`apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` + +##### 3.1.2 下载安装包 +获取 SeaTunnel Zeta 引擎服务安装包的另一种方式是从 https://seatunnel.apache.org/download 下载安装包并在线安装插件。 + +* 下载并安装连接器插件(这个过程中也会自动下载和安装一些第三方依赖包,比如 hadoop jar)。您可以参考 https://seatunnel.apache.org/docs/2.3.2/start-v2/locally/deployment 获取详细步骤。 +* 完成上述步骤后,您将获得一个可以用于在服务器上安装 SeaTunnel Zeta 引擎服务的安装包。运行 `tar -zcvf apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz apache-seatunnel-2.3.3-SNAPSHOT` + +##### 3.1.3 部署 SeaTunnel Zeta 服务端 + +在完成 3.1.1 或 3.1.2 后您可以获得一个安装包 `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz`,然后您可以将其复制到服务器节点并参考 https://seatunnel.apache.org/docs/seatunnel-engine/deployment 进行部署。 + +##### 3.1.4 在 SeaTunnel Web 运行节点部署 SeaTunnel Zeta 客户端 + +如果您使用 SeaTunnel Web,您需要在 SeaTunnel Web 运行节点部署一个 SeaTunnel Zeta 客户端。**如果 SeaTunnel Zeta 服务端和 SeaTunnel Web 在同一节点,您可以跳过此步骤**。 + +* 将 `apache-seatunnel-2.3.3-SNAPSHOT-bin.tar.gz` 复制到 SeaTunnel Web 节点并解压,解压**到与 SeaTunnel Zeta 服务端节点相同的路径下**。 +* 和 SeaTunnel Zeta 服务端节点相同,设置 `SEATUNNEL_HOME` 环境变量。 +* 参考 https://seatunnel.apache.org/docs/seatunnel-engine/deployment#6-config-seatunnel-engine-client 配置 `hazelcast-client.yaml`。 +* 运行 `$SEATUNNEL_HOME/bin/seatunnel.sh --config $SEATUNNEL_HOME/config/v2.batch.config.template`,如果该作业运行结束,表示客户端部署成功。 + +#### 3.2 从源码构建 SeaTunnel Web 安装包 + +``` +cd seatunnel-web +sh build.sh code +``` + +然后您可以在 `seatunnel-web/seatunnel-web-dist/target/apache-seatunnel-web-${project.version}.tar.gz` 目录中获得安装包。 + +#### 3.3 安装 + +将 `apache-seatunnel-web-${project.version}.tar.gz` 复制到服务器节点并解压。 + +```shell +tar -zxvf apache-seatunnel-web-${project.version}.tar.gz +```shell +tar -zxvf apache-seatunnel-web-${project.version}.tar.gz +``` + +#### 3.4 初始化数据库 + +1. 编辑 `apache-seatunnel-web-${project.version}/script/seatunnel_server_env.sh` 文件, 填写已安装的数据库 address, port, username, and password. 例如: + + ``` + export HOSTNAME="localhost" + export PORT="3306" + export USERNAME="root" + export PASSWORD="123456" + ``` +2. 运行初始化脚本 `sh apache-seatunnel-web-${project.version}/script/init_sql.sh` 如果操作过程中没有错误,表示初始化成功。 + +#### 3.5 配置应用并运行 SeaTunnel Web 后端服务 + +* 编辑 `apache-seatunnel-web-${project.version}/conf/application.yml` 在文件中填写数据库连接信息和数据服务接口相关信息。 +* 编辑 `apache-seatunnel-web-${project.version}/conf/application.yml` 文件,填写jwt.secretKey密钥,例如：SeaTunnel。 + +![image](docs/images/application_config.png) + +* 复制 `$SEATUNNEL_HOME/config/hazelcast-client.yaml` 到 `apache-seatunnel-web-${project.version}/conf/` +* 复制 `apache-seatunnel-2.3.3-SNAPSHOT/connectors/plugin-mapping.properties` 文件到 `apache-seatunnel-web-${project.version}/conf/` 目录. + +#### 3.6 运行 SeaTunnel Web + +```shell +cd apache-seatunnel-web-${project.version} +sh bin/seatunnel-backend-daemon.sh start +``` + +在浏览器中访问 http://127.0.0.1:8801/ui/ , 默认用户名和密码是 admin/admin. + +### 如何使用 + +完成所有前置工作后,我们可以打开以下网址: http://127.0.0.1:7890(请根据您的配置替换)来使用它。 + +现在,让我向您展示如何使用它。 + +#### 用户管理 + +![img.png](docs/images/UserImage.png) + +#### 任务管理 +![img.png](docs/images/TaskImage.png) + +#### 数据源管理 +![img.png](docs/images/DatasourceImage.png) + +#### 虚拟表 +![img.png](docs/images/VirtualImage.png) diff --git a/build.sh b/build.sh index da4f8b1df..cf2e47ae2 100644 --- a/build.sh +++ b/build.sh @@ -23,18 +23,22 @@ WORKDIR=$( pwd ) -DOCKER_VERSION=1.0.0-snapshot +if [ -z "$2" ]; then + VERSION="latest" +else + VERSION=$2 +fi # build code code() { - /bin/sh $WORKDIR/mvnw clean package -DskipTests + /bin/sh $WORKDIR/mvnw clean package -DskipTests -Pci # mv release zip - mv $WORKDIR/seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT.zip $WORKDIR/ + mv $WORKDIR/seatunnel-web-dist/target/apache-seatunnel-web-*.zip $WORKDIR/ } # build image image() { - docker buildx build --load --no-cache -t apache/seatunnel-web:$DOCKER_VERSION -t apache/seatunnel-web:latest -f $WORKDIR/docker/backend.dockerfile . + docker buildx build --load --no-cache -t apache/seatunnel-web:$VERSION -t apache/seatunnel-web:latest -f $WORKDIR/docker/backend.dockerfile . } # main @@ -46,7 +50,7 @@ case "$1" in image ;; *) - echo "Usage: seatunnel-daemon.sh {start|stop|status}" + echo "Usage: build.sh {code|image}" exit 1 ;; esac diff --git a/docs/QuickStart.md b/docs/QuickStart.md deleted file mode 100644 index 79dc6f485..000000000 --- a/docs/QuickStart.md +++ /dev/null @@ -1,86 +0,0 @@ -# 1.SeaTunnel 配置 - -## 1.1拉取SeaTunnel代码 - -github地址： https://github.com/apache/seatunnel - -## 1.2Install SeaTunnel - -1. 检查SeaTunnel version ,在根目录的pom文件中 - - ​ ![img.png](images\image-20230727135201238.png) - -2. 在终端中输入命令 ： ./mvnw clean install '-Dmaven.test.skip=true' '-Dcheckstyle.skip=true 或者输入 mvn clean install -DskipTests=true - -3. 执行完成后，显示如下： - -![img.png](images/image-20230727134919480.png) - -## 1.3启动SeaTunnel引擎 - -执行seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java文件，启动SeaTunnel引擎服务 - -# 2.SeaTunnel Web设置 - -## 2.1SeaTunnel依赖版本设置 - -在ST-Web跟目录的pom文件中设置SeaTunnel引擎版本，参考1.2中的SeaTunnel version - -​ ![img.png](images\image-20230727141624507.png) - -## 2.2添加 Connector依赖 - -假如你想使用Kafka作为数据源，需要在两个地方添加Kafka连接器的依赖 - -1. 在根目录的pom文件中的中添加kafka连接器依赖管理（注意修改该依赖的（不要是test级别）） - -​ ![img.png](images\image-20230727144108242.png) - -2. 在seatunnel-server/seatunnel-app/pom.xml文件中添加kafka连接器依赖（注意修改该依赖的（不要是test级别）） - -​ ![img.png](images\image-20230727144320362.png) - -## 2.3 打包SeaTunnel Web 项目 - -使用命令 mvn clean install -DskipTests=true - -## 2.4添加Enviroment variables - -1. 当打包完成之后，在文件seatunnel-web-dist/target/apache-seatunnel-web-1.0.0-SNAPSHOT/apache-seatunnel-web-1.0.0-SNAPSHOT（在第三步中会用到） - -2. 假如你使用的是IDEA，打开Edit Configurations - -​ ![img.png](images\image-20230727145045703.png) - -3. 添加Enviroment variables，将apache-seatunnel-web-1.0.0-SNAPSHOT文件的绝对路径添加的Enviroment variables中，命名为ST_WEB_BASEDIR_PATH - -![img.png](images\image-20230727145203278.png) - -​ ![img.png](images\image-20230727145452333.png) - -![image-20230727145652824](images\image-20230727145652824.png) - -4. 在SeatunnelApplication.java 下启动ST-Web - -# 3.QuickStart - -打开链接： http://127.0.0.1:5173/login - -初始用户名和密码都是admin - -​ ![img.png](images\image-20230727163613305.png) - -## 3.1创建数据源 - -​ ![image-20230727163756006](images\image-20230727163756006.png) - -选择一个你需要使用的数据源 - -![img.png](images\image-20230727163903835.png) - -![img.png](images\image-20230727164019922.png) - -## 3.2创建同步任务 - -![image-20230727164350427](images\image-20230727164350427.png) - diff --git a/docs/images/ds_create_project.png b/docs/images/ds_create_project.png deleted file mode 100644 index 71bb25168..000000000 Binary files a/docs/images/ds_create_project.png and /dev/null differ diff --git a/docs/images/ds_create_tenant.png b/docs/images/ds_create_tenant.png deleted file mode 100644 index 45dfe3363..000000000 Binary files a/docs/images/ds_create_tenant.png and /dev/null differ diff --git a/docs/images/ds_create_token.png b/docs/images/ds_create_token.png deleted file mode 100644 index 539c0bdf1..000000000 Binary files a/docs/images/ds_create_token.png and /dev/null differ diff --git a/docs/images/ds_create_user.png b/docs/images/ds_create_user.png deleted file mode 100644 index 5117f398b..000000000 Binary files a/docs/images/ds_create_user.png and /dev/null differ diff --git a/docs/images/idea_st_home.png b/docs/images/idea_st_home.png new file mode 100644 index 000000000..d4fd83493 Binary files /dev/null and b/docs/images/idea_st_home.png differ diff --git a/docs/images/image-20230727134919480.png b/docs/images/image-20230727134919480.png deleted file mode 100644 index 0851b5a30..000000000 Binary files a/docs/images/image-20230727134919480.png and /dev/null differ diff --git a/docs/images/image-20230727135201238.png b/docs/images/image-20230727135201238.png deleted file mode 100644 index 67ea06103..000000000 Binary files a/docs/images/image-20230727135201238.png and /dev/null differ diff --git a/docs/images/image-20230727135931257.png b/docs/images/image-20230727135931257.png deleted file mode 100644 index 2b08b0035..000000000 Binary files a/docs/images/image-20230727135931257.png and /dev/null differ diff --git a/docs/images/image-20230727140031522.png b/docs/images/image-20230727140031522.png deleted file mode 100644 index 9e5a708a8..000000000 Binary files a/docs/images/image-20230727140031522.png and /dev/null differ diff --git a/docs/images/image-20230727141624507.png b/docs/images/image-20230727141624507.png deleted file mode 100644 index 52a75a6fd..000000000 Binary files a/docs/images/image-20230727141624507.png and /dev/null differ diff --git a/docs/images/image-20230727144108242.png b/docs/images/image-20230727144108242.png deleted file mode 100644 index 261b948b6..000000000 Binary files a/docs/images/image-20230727144108242.png and /dev/null differ diff --git a/docs/images/image-20230727144320362.png b/docs/images/image-20230727144320362.png deleted file mode 100644 index df2a9177f..000000000 Binary files a/docs/images/image-20230727144320362.png and /dev/null differ diff --git a/docs/images/image-20230727145045703.png b/docs/images/image-20230727145045703.png deleted file mode 100644 index 2db92758d..000000000 Binary files a/docs/images/image-20230727145045703.png and /dev/null differ diff --git a/docs/images/image-20230727145203278.png b/docs/images/image-20230727145203278.png deleted file mode 100644 index 2cc4bfad0..000000000 Binary files a/docs/images/image-20230727145203278.png and /dev/null differ diff --git a/docs/images/image-20230727145452333.png b/docs/images/image-20230727145452333.png deleted file mode 100644 index d9558c764..000000000 Binary files a/docs/images/image-20230727145452333.png and /dev/null differ diff --git a/docs/images/image-20230727145652824.png b/docs/images/image-20230727145652824.png deleted file mode 100644 index cd48cf78e..000000000 Binary files a/docs/images/image-20230727145652824.png and /dev/null differ diff --git a/docs/images/image-20230727163613305.png b/docs/images/image-20230727163613305.png deleted file mode 100644 index c86296bc8..000000000 Binary files a/docs/images/image-20230727163613305.png and /dev/null differ diff --git a/docs/images/image-20230727163756006.png b/docs/images/image-20230727163756006.png deleted file mode 100644 index 4ddbdb4e1..000000000 Binary files a/docs/images/image-20230727163756006.png and /dev/null differ diff --git a/docs/images/image-20230727163903835.png b/docs/images/image-20230727163903835.png deleted file mode 100644 index 599d0c6fb..000000000 Binary files a/docs/images/image-20230727163903835.png and /dev/null differ diff --git a/docs/images/image-20230727164019922.png b/docs/images/image-20230727164019922.png deleted file mode 100644 index fc36bfa77..000000000 Binary files a/docs/images/image-20230727164019922.png and /dev/null differ diff --git a/docs/images/image-20230727164350427.png b/docs/images/image-20230727164350427.png deleted file mode 100644 index e77dd021b..000000000 Binary files a/docs/images/image-20230727164350427.png and /dev/null differ diff --git a/docs/images/st_web_basedir_path.png b/docs/images/st_web_basedir_path.png new file mode 100644 index 000000000..fff06c12b Binary files /dev/null and b/docs/images/st_web_basedir_path.png differ diff --git a/pom.xml b/pom.xml index adcde5485..c0e49c738 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 3.3.0 false - 2.12.7.1 + 2.12.6 2.12.7 8.0.16 2.1.214 @@ -71,11 +71,9 @@ 19.0 3.10.0 4.2.0 - 2.3.2 + 2.3.3 21.5.0.0 - 0.3.2-patch11 42.4.3 - 2.1.0.10 9.2.1.jre8 8.0.28 3.1.4 @@ -111,6 +109,7 @@ 1.2.17 2.17.1 1.7.25 + false @@ -256,6 +255,12 @@ org.apache.seatunnel seatunnel-common ${seatunnel-framework.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -275,6 +280,12 @@ org.apache.seatunnel seatunnel-api ${seatunnel-framework.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -294,6 +305,12 @@ org.apache.seatunnel seatunnel-plugin-discovery ${seatunnel-framework.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -412,33 +429,31 @@ ${jsoup.version} - + - org.apache.seatunnel - connector-common - ${seatunnel-framework.version} - test + cn.hutool + hutool-all + 5.8.16 - org.apache.seatunnel - connector-console - ${seatunnel-framework.version} - test + com.h2database + h2 + ${h2.version} + org.apache.seatunnel - connector-fake + connector-common ${seatunnel-framework.version} test - - - - org.apache.seatunnel - connector-jdbc - ${seatunnel-framework.version} - + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -482,17 +497,11 @@ ${checker.qual.version} - - org.awaitility - awaitility - ${awaitility.version} - test - - org.apache.seatunnel seatunnel-hadoop3-3.1.4-uber ${hadoop-uber.version} + provided org.apache.avro @@ -545,7 +554,7 @@ org.apache.seatunnel datasource-s3 ${project.version} - provided + test @@ -556,23 +565,23 @@ - org.apache.seatunnel - connector-common - ${seatunnel-framework.version} + org.awaitility + awaitility + ${awaitility.version} test - - org.apache.seatunnel - seatunnel-transforms-v2 - ${seatunnel-framework.version} - - org.apache.seatunnel connector-console ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -580,300 +589,602 @@ connector-fake ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-kafka ${seatunnel-framework.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-base ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-feishu ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-wechat ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-myhours ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-lemlist ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-klaviyo ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-onesignal ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-notion ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-jdbc ${seatunnel-framework.version} - + test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-socket ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-clickhouse ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-pulsar ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-hive ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-hadoop ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-local ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-oss ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-oss-jindo ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-ftp ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-sftp ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-hudi ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-dingtalk ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-kudu ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-email ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-elasticsearch ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-iotdb ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.seatunnel connector-neo4j ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-redis ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-google-sheets ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-datahub ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-sentry ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-mongodb ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-iceberg ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-influxdb ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-cassandra ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-file-s3 ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-amazondynamodb ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-starrocks ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-tablestore ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-slack ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-gitlab ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-http-jira ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-rabbitmq ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-openmldb ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-doris ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-maxcompute ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-cdc-mysql ${seatunnel-framework.version} - + test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-cdc-sqlserver ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.seatunnel connector-cdc-oracle ${seatunnel-framework.version} test + + + org.apache.logging.log4j + log4j-slf4j-impl + + @@ -885,18 +1196,6 @@ commons-lang3 ${commons-lang3.version} - - org.apache.seatunnel - connector-cdc-sqlserver - ${seatunnel-framework.version} - test - - - org.apache.seatunnel - connector-cdc-mysql - ${seatunnel-framework.version} - - org.projectlombok lombok @@ -1134,6 +1433,7 @@ spotless-maven-plugin ${spotless.version} + ${skip.spotless} 1.7 @@ -1217,7 +1517,6 @@ org.apache.maven.plugins maven-release-plugin - 3.0.1 true @{project.version} @@ -1237,11 +1536,6 @@ maven-surefire-plugin - - org.apache.maven.plugins - maven-checkstyle-plugin - - org.codehaus.mojo license-maven-plugin @@ -1254,7 +1548,7 @@ - https://github.com/apache/incubator-seatunnel + https://github.com/apache/seatunnel-web @@ -1279,15 +1573,15 @@ - scm:git:https://github.com/apache/incubator-seatunnel.git - scm:git:https://github.com/apache/incubator-seatunnel.git - https://github.com/apache/incubator-seatunnel + scm:git:https://github.com/apache/seatunnel-web.git + scm:git:https://github.com/apache/seatunnel-web.git + https://github.com/apache/seatunnel-web HEAD GitHub - https://github.com/apache/incubator-seatunnel/issues + https://github.com/apache/seatunnel/issues diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java index 9c629859f..34fb2850e 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java @@ -67,11 +67,11 @@ protected AbstractDataSourceClient() { for (String pluginName : DatasourceLoadConfig.pluginSet) { log.info("plugin set : " + pluginName); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(getCustomClassloader(pluginName)); if (DatasourceLoadConfig.classLoaderChannel.get(pluginName.toUpperCase()) != null) { log.info(pluginName + " is exist"); continue; } + Thread.currentThread().setContextClassLoader(getCustomClassloader(pluginName)); try { Class clazz = Class.forName( @@ -81,7 +81,7 @@ protected AbstractDataSourceClient() { Thread.currentThread().getContextClassLoader()); DataSourceFactory factory = (DataSourceFactory) clazz.getDeclaredConstructor().newInstance(); - log.info("factory : " + String.valueOf(factory)); + log.info("factory : " + factory); Set dataSourcePluginInfos = factory.supportedDataSources(); dataSourcePluginInfos.forEach( dataSourceInfo -> { @@ -126,11 +126,7 @@ public List listAllDataSources() { protected DataSourceChannel getDataSourceChannel(String pluginName) { checkNotNull(pluginName, "pluginName cannot be null"); - // Integer index = supportedDataSourceIndex.get(pluginName.toUpperCase()); - // if (index == null) { - // throw new DataSourceSDKException( - // "The %s plugin is not supported or plugin not exist.", pluginName); - // } + return DatasourceLoadConfig.classLoaderChannel.get(pluginName.toUpperCase()); } @@ -229,7 +225,6 @@ private ClassLoader getCustomClassloader(String pluginName) { log.info("ST_WEB_BASEDIR_PATH is : " + getenv); String libPath = StringUtils.isEmpty(getenv) ? "/datasource" : (getenv + "/datasource"); - // String libPath = "/root/apache-seatunnel-web-2.4.7-WS-SNAPSHOT/datasource/"; File jarDirectory = new File(libPath); File[] jarFiles = jarDirectory.listFiles( @@ -249,7 +244,11 @@ private ClassLoader getCustomClassloader(String pluginName) { }); log.info("jar file length :" + (jarFiles == null ? 0 : jarFiles.length)); - log.info("jar file length :" + (jarFiles == null ? 0 : jarFiles[0].getName())); + log.info( + "jar file name :" + + (jarFiles == null + ? 0 + : jarFiles.length == 0 ? "no jar" : jarFiles[0].getName())); DatasourceClassLoader customClassLoader = DatasourceLoadConfig.datasourceClassLoaders.get(pluginName.toUpperCase()); try { @@ -272,7 +271,7 @@ private ClassLoader getCustomClassloader(String pluginName) { } catch (Exception e) { e.printStackTrace(); } - log.info("custom loader is:" + String.valueOf(customClassLoader)); + log.info("custom loader is:" + customClassLoader); return customClassLoader; } diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java index f29f29c9f..9ef3f3a7f 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceClassLoader.java @@ -65,7 +65,6 @@ public Class loadClass(String name) throws ClassNotFoundException { @Override public URL getResource(String name) { - log.info("getResource : " + name); // first, try and find it via the URLClassloader URL urlClassLoaderResource = findResource(name); if (urlClassLoaderResource != null) { @@ -79,7 +78,6 @@ public URL getResource(String name) { @Override public Enumeration getResources(String name) throws IOException { // first get resources from URLClassloader - log.info("getResources : " + name); Enumeration urlClassLoaderResources = findResources(name); final List result = new ArrayList<>(); diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java index d169c310b..b2869691a 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java @@ -57,8 +57,7 @@ public class DatasourceLoadConfig { "JDBC-CLICKHOUSE", "org.apache.seatunnel.datasource.plugin.clickhouse.jdbc.ClickhouseJdbcDataSourceFactory"); classLoaderFactoryName.put( - "HIVE", - "org.apache.seatunnel.datasource.plugin.hive.jdbc.HiveJdbcDataSourceFactory"); + "HIVE", "com.whaleops.datasource.datasource.plugin.hive.HiveDataSourceFactory"); classLoaderFactoryName.put( "JDBC-ORACLE", "org.apache.seatunnel.datasource.plugin.oracle.jdbc.OracleJdbcDataSourceFactory"); @@ -93,6 +92,11 @@ public class DatasourceLoadConfig { classLoaderFactoryName.put( "JDBC-STARROCKS", "org.apache.seatunnel.datasource.plugin.starrocks.jdbc.StarRocksJdbcDataSourceFactory"); + classLoaderFactoryName.put( + "MONGODB", "com.apache.seatunnel.datasource.plugin.mongodb.MongoDataSourceFactory"); + classLoaderFactoryName.put( + "JDBC-DB2", + "org.apache.seatunnel.datasource.plugin.db2.jdbc.Db2JdbcDataSourceFactory"); classLoaderJarName.put("JDBC-ORACLE", "datasource-jdbc-oracle-"); classLoaderJarName.put("JDBC-CLICKHOUSE", "datasource-jdbc-clickhouse-"); @@ -107,11 +111,13 @@ public class DatasourceLoadConfig { classLoaderJarName.put("ELASTICSEARCH", "datasource-elasticsearch-"); classLoaderJarName.put("S3", "datasource-s3-"); - classLoaderJarName.put("HIVE", "datasource-jdbc-hive-"); + classLoaderJarName.put("HIVE", "datasource-hive-"); classLoaderJarName.put("KAFKA", "datasource-kafka-"); classLoaderJarName.put("STARROCKS", "datasource-starrocks-"); classLoaderJarName.put("S3-REDSHIFT", "datasource-s3redshift-"); classLoaderJarName.put("JDBC-STARROCKS", "datasource-jdbc-starrocks-"); + classLoaderJarName.put("MONGODB", "datasource-mongodb-"); + classLoaderJarName.put("JDBC-DB2", "datasource-jdbc-db2-"); } public static final Set pluginSet = @@ -122,16 +128,15 @@ public class DatasourceLoadConfig { "Hive", "JDBC-Oracle", "JDBC-Postgres", - "JDBC-Redshift", "JDBC-SQLServer", "JDBC-TiDB", "Kafka", "MySQL-CDC", "S3", - "S3-Redshift", "SqlServer-CDC", - "JDBC-StarRocks", - "StarRocks"); + "StarRocks", + "MongoDB", + "JDBC-Db2"); public static Map datasourceClassLoaders = new HashMap<>(); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml index 2eac02b34..d5dee4503 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-all/pom.xml @@ -55,11 +55,6 @@ datasource-jdbc-tidb ${project.version} - - org.apache.seatunnel - datasource-jdbc-redshift - ${project.version} - org.apache.seatunnel datasource-jdbc-sqlserver @@ -82,11 +77,6 @@ datasource-elasticsearch ${project.version} - - org.apache.seatunnel - datasource-s3redshift - ${project.version} - org.apache.seatunnel datasource-starrocks @@ -107,6 +97,11 @@ datasource-sqlserver-cdc ${project.version} + + org.apache.seatunnel + datasource-jdbc-db2 + ${project.version} + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml index 017a221ad..61b995fde 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/pom.xml @@ -61,4 +61,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml new file mode 100644 index 000000000..672885cc9 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/pom.xml @@ -0,0 +1,114 @@ + + + + 4.0.0 + + org.apache.seatunnel + seatunnel-datasource-plugins + ${revision} + + + datasource-hive + + + 3.1.3 + 24.1-jre + + + + + org.apache.seatunnel + datasource-plugins-api + ${project.version} + provided + + + org.apache.seatunnel + seatunnel-hadoop3-3.1.4-uber + 2.3.2 + optional + + + org.apache.avro + avro + + + + + org.apache.hive + hive-exec + ${hive.exec.version} + + + log4j + log4j + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-web + + + org.slf4j + slf4j-log4j12 + + + org.apache.parquet + parquet-hadoop-bundle + + + jdk.tools + jdk.tools + + + org.pentaho + pentaho-aggdesigner-algorithm + + + org.apache.avro + avro + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java new file mode 100644 index 000000000..e9826c59a --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.whaleops.datasource.datasource.plugin.hive; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; +import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; + +import com.google.common.collect.Sets; + +import java.util.Set; + +public class HiveConfig { + public static final String PLUGIN_NAME = "Hive"; + + public static final DataSourcePluginInfo HIVE_DATASOURCE_PLUGIN_INFO = + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); + + public static final Set HIVE_SYSTEM_DATABASES = Sets.newHashSet(); + + public static final OptionRule OPTION_RULE = + OptionRule.builder().required(HiveOptionRule.METASTORE_URI).build(); +} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java similarity index 70% rename from seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java index 658e59b29..d469b3b1b 100644 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ExecuteDto.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveConstants.java @@ -15,17 +15,15 @@ * limitations under the License. */ -package org.apache.seatunnel.spi.scheduler.dto; +package com.whaleops.datasource.datasource.plugin.hive; -import org.apache.seatunnel.spi.scheduler.enums.ExecuteTypeEnum; +import com.google.common.collect.Sets; -import lombok.Builder; -import lombok.Data; +import java.util.Set; -@Data -@Builder -public class ExecuteDto { - private JobDto jobDto; - private ExecuteTypeEnum executeTypeEnum; - private ComplementDataDto complementDataDto; +public class HiveConstants { + + public static final Set HIVE_SYSTEM_DATABASES = + Sets.newHashSet( + "information_schema", "mysql", "performance_schema", "sys", "test", "hivedb"); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java new file mode 100644 index 000000000..cebdbf8e8 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.whaleops.datasource.datasource.plugin.hive; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang.StringUtils; + +import com.whaleops.datasource.datasource.plugin.hive.client.HiveClient; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public class HiveDataSourceChannel implements DataSourceChannel { + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return HiveOptionRule.optionRule(); + } + + @Override + public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + return HiveOptionRule.metadataRule(); + } + + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + + try (HiveClient hiveClient = HiveClient.createInstance(requestParams)) { + String size = options.get("size"); + Integer sizeInt = StringUtils.isNotEmpty(size) ? Integer.parseInt(size) : 0; + return hiveClient.getAllTables(database, options.get("filterName"), sizeInt); + } + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + try (HiveClient hiveClient = HiveClient.createInstance(requestParams); ) { + return hiveClient.getAllDatabases(); + } + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + return checkHiveConnectivity(requestParams); + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + return getTableFields(requestParams, database, table); + } + + @Override + public Map> getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { + Map> tableFields = new HashMap<>(tables.size()); + for (String table : tables) { + tableFields.put(table, getTableFields(requestParams, database, table)); + } + return tableFields; + } + + protected boolean checkHiveConnectivity(Map requestParams) { + try (HiveClient ignored = HiveClient.createInstance(requestParams)) { + System.out.println(ignored.getAllDatabases()); + return true; + } catch (Exception e) { + throw new DataSourcePluginException( + "check hive connectivity failed, " + e.getMessage(), e); + } + } + + protected List getTableFields( + Map requestParams, String dbName, String tableName) { + try (HiveClient hiveClient = HiveClient.createInstance(requestParams)) { + return hiveClient.getFields(dbName, tableName); + } + } + + private static boolean checkHostConnectable(String host, int port) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 1000); + return true; + } catch (IOException e) { + return false; + } + } + + private boolean isNotSystemDatabase(String pluginName, String dbName) { + // FIXME,filters system databases + return true; + } + + private boolean convertToBoolean(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } + if (value instanceof String) { + return value.equals("TRUE"); + } + return false; + } +} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java similarity index 52% rename from seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java index e4d2f08a5..712364532 100644 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/ReleaseStateEnum.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceFactory.java @@ -15,32 +15,31 @@ * limitations under the License. */ -package org.apache.seatunnel.scheduler.dolphinscheduler.enums; +package com.whaleops.datasource.datasource.plugin.hive; -import org.apache.seatunnel.server.common.SeatunnelErrorEnum; -import org.apache.seatunnel.server.common.SeatunnelException; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; -public enum ReleaseStateEnum { - ONLINE(true), - OFFLINE(false), - ; +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; - private final boolean publish; +import java.util.Set; - ReleaseStateEnum(boolean publish) { - this.publish = publish; +@Slf4j +public class HiveDataSourceFactory implements DataSourceFactory { + @Override + public String factoryIdentifier() { + return "Hive"; } - public static ReleaseStateEnum parse(String name) { - for (ReleaseStateEnum value : values()) { - if (value.name().equalsIgnoreCase(name)) { - return value; - } - } - throw new SeatunnelException(SeatunnelErrorEnum.NO_SUCH_ELEMENT); + @Override + public Set supportedDataSources() { + return Sets.newHashSet(HiveConfig.HIVE_DATASOURCE_PLUGIN_INFO); } - public boolean isPublish() { - return publish; + @Override + public DataSourceChannel createChannel() { + return new HiveDataSourceChannel(); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java new file mode 100644 index 000000000..dcea7d2fa --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveOptionRule.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.whaleops.datasource.datasource.plugin.hive; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.util.OptionRule; + +public class HiveOptionRule { + + public static final Option METASTORE_URI = + Options.key("metastore_uri") + .stringType() + .noDefaultValue() + .withDescription("thrift url, eg:" + "thrift://127.0.0.1:9083"); + + public static final Option KERBEROS_PRINCIPAL = + Options.key("kerberos_principal") + .stringType() + .noDefaultValue() + .withDescription("jdbc kerberos_principal"); + + public static final Option KERBEROS_KEYTAB_PATH = + Options.key("kerberos_keytab_path") + .stringType() + .noDefaultValue() + .withDescription("jdbc kerberos_keytab_path"); + + public static final Option KERBEROS_KRB5_CONF_PATH = + Options.key("kerberos_krb5_conf_path") + .stringType() + .noDefaultValue() + .withDescription("jdbc kerberos_keytab_path"); + + public static final Option HDFS_SITE_PATH = + Options.key("hdfs_site_path") + .stringType() + .noDefaultValue() + .withDescription("jdbc hdfs_site_path"); + + public static final Option HIVE_SITE_PATH = + Options.key("hive_site_path") + .stringType() + .noDefaultValue() + .withDescription("jdbc hive_site_path"); + + public static OptionRule optionRule() { + return OptionRule.builder() + .required(METASTORE_URI) + .optional(KERBEROS_PRINCIPAL) + .optional(KERBEROS_KRB5_CONF_PATH) + .optional(KERBEROS_KEYTAB_PATH) + .optional(HDFS_SITE_PATH) + .optional(HIVE_SITE_PATH) + .build(); + } + + public static OptionRule metadataRule() { + // todo + return OptionRule.builder().build(); + } + + public static final Option TABLE_NAME = + Options.key("table").stringType().noDefaultValue().withDescription("hive table"); +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java new file mode 100644 index 000000000..92bf0e38c --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/client/HiveClient.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.whaleops.datasource.datasource.plugin.hive.client; + +import org.apache.seatunnel.common.utils.ExceptionUtils; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.thrift.TException; + +import com.whaleops.datasource.datasource.plugin.hive.HiveOptionRule; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; + +@Slf4j +public class HiveClient implements AutoCloseable { + + private final HiveMetaStoreClient hiveMetaStoreClient; + + public HiveClient(HiveMetaStoreClient hiveMetaStoreClient) { + this.hiveMetaStoreClient = hiveMetaStoreClient; + } + + public static HiveClient createInstance(Map reqParam) { + checkNotNull( + reqParam.get(HiveOptionRule.METASTORE_URI.key()), + "hive metastore_uri cannot be null"); + String metastoreUri = reqParam.get(HiveOptionRule.METASTORE_URI.key()); + String kerberosPrincipal = reqParam.get(HiveOptionRule.KERBEROS_PRINCIPAL.key()); + String kerberosKrb5ConfPath = reqParam.get(HiveOptionRule.KERBEROS_KRB5_CONF_PATH.key()); + String kerberosKeytabPath = reqParam.get(HiveOptionRule.KERBEROS_KEYTAB_PATH.key()); + String hdfsSitePath = reqParam.get(HiveOptionRule.HDFS_SITE_PATH.key()); + String hiveSitePath = reqParam.get(HiveOptionRule.HIVE_SITE_PATH.key()); + System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath); + System.setProperty("krb.principal", "hadoop"); + try { + if (StringUtils.isNotEmpty(kerberosPrincipal)) { + // login Kerberos + Configuration configuration = new Configuration(); + if (StringUtils.isNotEmpty(kerberosPrincipal)) { + configuration.addResource(new File(hdfsSitePath).toURI().toURL()); + } + doKerberosAuthentication(configuration, kerberosPrincipal, kerberosKeytabPath); + } + HiveConf hiveConf = new HiveConf(); + hiveConf.set("hive.metastore.uris", metastoreUri); + if (StringUtils.isNotEmpty(hiveSitePath)) { + hiveConf.addResource(new File(hiveSitePath).toURI().toURL()); + } + log.info("hive client conf:{}", hiveConf); + + return new HiveClient(new HiveMetaStoreClient(hiveConf)); + } catch (Exception e) { + String errorMsg = + String.format( + "Using this hive uris [%s] to initialize " + + "hive metastore client instance failed", + metastoreUri); + log.error(ExceptionUtils.getMessage(e)); + throw new DataSourcePluginException(errorMsg, e); + } + } + + /*private static void authKerberos( + String kerberosKrb5ConfPath, String kerberosKeytabPath, String kerberosPrincipal) + throws IOException { + System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath); + Configuration configuration = new Configuration(); + configuration.set("hadoop.security.authentication", "Kerberos"); + configuration.setBoolean("hadoop.security.authorization", true); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabPath); + }*/ + + public static void doKerberosAuthentication( + Configuration configuration, String principal, String keytabPath) { + if (StringUtils.isBlank(principal) || StringUtils.isBlank(keytabPath)) { + log.warn( + "Principal [{}] or keytabPath [{}] is empty, it will skip kerberos authentication", + principal, + keytabPath); + } else { + configuration.set("hadoop.security.authentication", "kerberos"); + UserGroupInformation.setConfiguration(configuration); + try { + log.info( + "Start Kerberos authentication using principal {} and keytab {}", + principal, + keytabPath); + UserGroupInformation.loginUserFromKeytab(principal, keytabPath); + log.info("Kerberos authentication successful"); + } catch (IOException e) { + throw new DataSourcePluginException( + "check hive connectivity failed, " + e.getMessage(), e); + } + } + } + + @Override + public void close() { + try { + if (hiveMetaStoreClient != null) { + hiveMetaStoreClient.close(); + } + } catch (Exception e) { + log.warn("close hive connection error", e); + } + } + + public List getAllDatabases() { + try { + return hiveMetaStoreClient.getAllDatabases(); + } catch (Exception e) { + log.error(ExceptionUtils.getMessage(e)); + throw new DataSourcePluginException("get database names failed", e); + } + } + + public List getAllTables(String dbName) { + return getAllTables(dbName, null, null); + } + + public List getAllTables(String dbName, String filterName, Integer size) { + try { + + List tables = hiveMetaStoreClient.getTables(dbName, "*"); + List filteredTables = new ArrayList<>(); + // hive api whether to support filtering directly + if (StringUtils.isNotEmpty(filterName)) { + for (String tableName : tables) { + if (tableName.contains(filterName)) { + filteredTables.add(tableName); + } + } + } else { + filteredTables.addAll(tables); + } + // filter Iceberg table + this.filteredIcebergTable(filteredTables, dbName); + if (size != null && size > 0) { + return filteredTables.subList(0, Math.min(size, filteredTables.size())); + } + return filteredTables; + } catch (Exception e) { + log.error(ExceptionUtils.getMessage(e)); + throw new DataSourcePluginException("get table names failed", e); + } + } + + // 过滤掉 Iceberg 表 + private void filteredIcebergTable(List filteredTables, String dbName) + throws TException { + if (CollectionUtils.isNotEmpty(filteredTables)) { + Iterator iterator = filteredTables.iterator(); + while (iterator.hasNext()) { + String element = iterator.next(); + if (isIcebergTable(dbName, element)) { + iterator.remove(); + } + } + } + } + + // 判断是否为 Iceberg 表 + private boolean isIcebergTable(String dbName, String tableName) throws TException { + Table table = hiveMetaStoreClient.getTable(dbName, tableName); + Map parameters = table.getParameters(); + String tableType = parameters.get("table_type"); + return "ICEBERG".equals(tableType); + } + + public List getFields(String dbName, String tableName) { + List tableFields = new ArrayList<>(); + try { + List fields = hiveMetaStoreClient.getFields(dbName, tableName); + fields.forEach( + field -> { + TableField tableField = new TableField(); + String columnName = field.getName(); + tableField.setPrimaryKey(false); + tableField.setName(columnName); + String stringType = field.getType(); + String fieldTypeStart = stringType.split("[(<]")[0]; + tableField.setType(fieldTypeStart); + tableField.setComment(field.getComment()); + tableFields.add(tableField); + }); + return tableFields; + } catch (TException e) { + log.error(ExceptionUtils.getMessage(e)); + throw new DataSourcePluginException("get table fields failed", e); + } + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml index dcc91da43..31f722bd5 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/pom.xml @@ -55,8 +55,25 @@ com.clickhouse clickhouse-jdbc ${clickhouse.version} - provided + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml similarity index 52% rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml index 61b84cb8b..a13423735 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/pom.xml @@ -22,7 +22,11 @@ ${revision} - datasource-s3-redshift + datasource-jdbc-db2 + + + 8.0.28 + @@ -31,39 +35,58 @@ ${project.version} provided + + org.apache.commons + commons-lang3 + + + + com.google.auto.service + auto-service + org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber - 2.4.5-WS-SNAPSHOT - - - org.apache.avro - avro - - + seatunnel-api + provided + + - com.amazon.redshift - redshift-jdbc42 - ${redshift.version} + mysql + mysql-connector-java + ${mysql-connector.version} provided + - org.apache.hadoop - hadoop-aws - ${hadoop-aws.version} - - - jdk.tools - jdk.tools - - + com.ibm.db2.jcc + db2jcc + db2jcc4 - com.amazonaws - aws-java-sdk-bundle - ${aws-java-sdk-bundle.version} + org.apache.seatunnel + seatunnel-api + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java new file mode 100644 index 000000000..90673587c --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2DataSourceConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.datasource.plugin.db2.jdbc; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; +import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; + +import com.google.common.collect.Sets; + +import java.util.Set; + +public class Db2DataSourceConfig { + + public static final String PLUGIN_NAME = "JDBC-Db2"; + + public static final DataSourcePluginInfo DB2_DATASOURCE_PLUGIN_INFO = + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); + + public static final Set DB2_SYSTEM_DATABASES = + Sets.newHashSet("information_schema", "mysql", "performance_schema", "sys"); + + public static final OptionRule OPTION_RULE = + OptionRule.builder() + .required(Db2OptionRule.URL, Db2OptionRule.DRIVER) + .optional(Db2OptionRule.USER, Db2OptionRule.PASSWORD) + .build(); + + public static final OptionRule METADATA_RULE = + OptionRule.builder().required(Db2OptionRule.DATABASE, Db2OptionRule.TABLE).build(); +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java new file mode 100644 index 000000000..844a47563 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceChannel.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.datasource.plugin.db2.jdbc; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang3.StringUtils; + +import lombok.NonNull; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class Db2JdbcDataSourceChannel implements DataSourceChannel { + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return Db2DataSourceConfig.OPTION_RULE; + } + + @Override + public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + return Db2DataSourceConfig.METADATA_RULE; + } + + @Override + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + List tableNames = new ArrayList<>(); + String filterName = options.get("filterName"); + String size = options.get("size"); + boolean isSize = StringUtils.isNotEmpty(size); + if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) { + filterName = "%" + filterName + "%"; + } else if (StringUtils.equals(filterName, "")) { + filterName = null; + } + try (Connection connection = getConnection(requestParams); + ResultSet resultSet = + connection + .getMetaData() + .getTables(null, null, "%", new String[] {"TABLE"})) { + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + if (StringUtils.isNotBlank(tableName)) { + tableNames.add(tableName); + if (isSize && tableNames.size() >= Integer.parseInt(size)) { + break; + } + } + } + return tableNames; + } catch (ClassNotFoundException | SQLException e) { + throw new DataSourcePluginException("get table names failed", e); + } + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + // Hardcoded list of example database names + List dbNames = Arrays.asList("default"); + return dbNames; + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + try (Connection ignored = getConnection(requestParams)) { + return true; + } catch (Exception e) { + throw new DataSourcePluginException("check jdbc connectivity failed", e); + } + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + List tableFields = new ArrayList<>(); + try (Connection connection = getConnection(requestParams)) { + DatabaseMetaData metaData = connection.getMetaData(); + + // Retrieve primary key information + String primaryKey = getPrimaryKey(metaData, database, table); + + // Retrieve column information + try (ResultSet resultSet = metaData.getColumns(null, null, table, null)) { + + while (resultSet.next()) { + TableField tableField = new TableField(); + String columnName = resultSet.getString("COLUMN_NAME"); + + // Set primary key flag + tableField.setPrimaryKey(primaryKey != null && primaryKey.equals(columnName)); + + // Set other field attributes + tableField.setName(columnName); + tableField.setType(resultSet.getString("TYPE_NAME")); + tableField.setComment(resultSet.getString("REMARKS")); + + // Set nullable flag + String isNullable = resultSet.getString("IS_NULLABLE"); + tableField.setNullable("YES".equalsIgnoreCase(isNullable)); + + tableFields.add(tableField); + } + } + } catch (SQLException e) { + // Log the exception and rethrow as DataSourcePluginException + System.out.println("Error while retrieving table fields: " + e); + throw new DataSourcePluginException("Failed to get table fields", e); + } catch (ClassNotFoundException e) { + // Log the exception and rethrow as DataSourcePluginException + System.out.println("JDBC driver class not found" + e); + throw new DataSourcePluginException("JDBC driver class not found", e); + } + return tableFields; + } + + @Override + public Map> getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull List tables) { + return tables.parallelStream() + .collect( + Collectors.toMap( + Function.identity(), + table -> + getTableFields( + pluginName, requestParams, database, table))); + } + + private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) + throws SQLException { + ResultSet primaryKeysInfo = metaData.getPrimaryKeys(null, dbName, tableName); + while (primaryKeysInfo.next()) { + return primaryKeysInfo.getString("COLUMN_NAME"); + } + return null; + } + + private Connection getConnection(Map requestParams) + throws SQLException, ClassNotFoundException { + // Ensure the DB2 JDBC driver is loaded + Class.forName("com.ibm.db2.jcc.DB2Driver"); + checkNotNull(requestParams.get(Db2OptionRule.URL.key()), "Jdbc url cannot be null"); + String url = requestParams.get(Db2OptionRule.URL.key()); + if (requestParams.containsKey(Db2OptionRule.USER.key())) { + String username = requestParams.get(Db2OptionRule.USER.key()); + String password = requestParams.get(Db2OptionRule.PASSWORD.key()); + return DriverManager.getConnection(url, username, password); + } + return DriverManager.getConnection(url); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java similarity index 63% rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java index ee6aa4ba0..afdd75c95 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2JdbcDataSourceFactory.java @@ -15,42 +15,34 @@ * limitations under the License. */ -package org.apache.seatunnel.datasource.plugin.redshift.s3; +package org.apache.seatunnel.datasource.plugin.db2.jdbc; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; -import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; import com.google.auto.service.AutoService; import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; import java.util.Set; +@Slf4j @AutoService(DataSourceFactory.class) -public class S3RedshiftDataSourceFactory implements DataSourceFactory { +public class Db2JdbcDataSourceFactory implements DataSourceFactory { + @Override public String factoryIdentifier() { - return "S3-Redshift"; + return Db2DataSourceConfig.PLUGIN_NAME; } @Override public Set supportedDataSources() { - DataSourcePluginInfo s3DatasourcePluginInfo = - DataSourcePluginInfo.builder() - .name("S3-Redshift") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .version("1.0.0") - .supportVirtualTables(false) - .icon("S3-Redshift") - .icon("S3-Redshift") - .build(); - - return Sets.newHashSet(s3DatasourcePluginInfo); + return Sets.newHashSet(Db2DataSourceConfig.DB2_DATASOURCE_PLUGIN_INFO); } @Override public DataSourceChannel createChannel() { - return new S3RedshiftDataSourceChannel(); + return new Db2JdbcDataSourceChannel(); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java new file mode 100644 index 000000000..e5e2b9b1b --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-db2/src/main/java/org/apache/seatunnel/datasource/plugin/db2/jdbc/Db2OptionRule.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.datasource.plugin.db2.jdbc; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; + +public class Db2OptionRule { + + public static final Option URL = + Options.key("url") + .stringType() + .noDefaultValue() + .withDescription( + "jdbc url, eg:" + + " jdbc:mysql://localhost:3306/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); + + public static final Option USER = + Options.key("user").stringType().noDefaultValue().withDescription("jdbc user"); + + public static final Option PASSWORD = + Options.key("password").stringType().noDefaultValue().withDescription("jdbc password"); + + public static final Option DATABASE = + Options.key("database").stringType().noDefaultValue().withDescription("jdbc database"); + + public static final Option TABLE = + Options.key("table").stringType().noDefaultValue().withDescription("jdbc table"); + + public static final Option DRIVER = + Options.key("driver") + .enumType(DriverType.class) + .defaultValue(DriverType.DB2) + .withDescription("driver"); + + public enum DriverType { + DB2("com.ibm.db2.jcc.DB2Driver"), + ; + private final String driverClassName; + + DriverType(String driverClassName) { + this.driverClassName = driverClassName; + } + + public String getDriverClassName() { + return driverClassName; + } + + @Override + public String toString() { + return driverClassName; + } + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml index 32dce470d..f24fdb7c3 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/pom.xml @@ -33,4 +33,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml index 479fc5984..8a1e0b6a7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/pom.xml @@ -60,4 +60,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java index fe7842f6b..24e863b28 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,17 +59,28 @@ public List getTables( @NonNull String pluginName, Map requestParams, String database, - Map option) { + Map options) { List tableNames = new ArrayList<>(); + String filterName = options.get("filterName"); + String size = options.get("size"); + boolean isSize = StringUtils.isNotEmpty(size); + if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) { + filterName = "%" + filterName + "%"; + } else if (StringUtils.equals(filterName, "")) { + filterName = null; + } try (Connection connection = getConnection(requestParams); ResultSet resultSet = connection .getMetaData() - .getTables(database, null, null, new String[] {"TABLE"})) { + .getTables(database, null, filterName, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { tableNames.add(tableName); + if (isSize && tableNames.size() >= Integer.parseInt(size)) { + break; + } } } return tableNames; @@ -176,11 +188,15 @@ private Connection getConnection(Map requestParams, String datab String url = JdbcUtils.replaceDatabase( requestParams.get(MysqlOptionRule.URL.key()), databaseName); + + Properties info = new java.util.Properties(); + info.put("autoDeserialize", "false"); + info.put("allowLoadLocalInfile", "false"); + info.put("allowLoadLocalInfileInPath", ""); if (requestParams.containsKey(MysqlOptionRule.USER.key())) { - String username = requestParams.get(MysqlOptionRule.USER.key()); - String password = requestParams.get(MysqlOptionRule.PASSWORD.key()); - return DriverManager.getConnection(url, username, password); + info.put("user", requestParams.get(MysqlOptionRule.USER.key())); + info.put("password", requestParams.get(MysqlOptionRule.PASSWORD.key())); } - return DriverManager.getConnection(url); + return DriverManager.getConnection(url, info); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml index 9e108276c..287b8f75c 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/pom.xml @@ -59,4 +59,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java index 6844cd3a4..775decb1c 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java @@ -26,6 +26,7 @@ import org.apache.commons.lang3.StringUtils; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -33,12 +34,15 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; +@Slf4j public class OracleDataSourceChannel implements DataSourceChannel { @Override @@ -56,18 +60,72 @@ public List getTables( @NonNull String pluginName, Map requestParams, String database, - Map option) { + Map options) { + StringBuilder sqlWhere = new StringBuilder(); + final String sql = + "SELECT * FROM ( SELECT OWNER, TABLE_NAME FROM ALL_TABLES\n" + + "WHERE TABLE_NAME NOT LIKE 'MDRT_%'\n" + + " AND TABLE_NAME NOT LIKE 'MDRS_%'\n" + + " AND TABLE_NAME NOT LIKE 'MDXT_%'\n" + + " AND (TABLE_NAME NOT LIKE 'SYS_IOT_OVER_%' AND IOT_NAME IS NULL)" + + "AND OWNER NOT IN ('APPQOSSYS', 'AUDSYS', 'CTXSYS', 'DVSYS', 'DBSFWUSER', 'DBSNMP',\n" + + " 'GSMADMIN_INTERNAL', 'LBACSYS', 'MDSYS', 'OJVMSYS', 'OLAPSYS',\n" + + " 'ORDDATA', 'ORDSYS', 'OUTLN', 'SYS', 'SYSTEM', 'WMSYS',\n" + + " 'XDB', 'EXFSYS', 'SYSMAN')"; + sqlWhere.append(sql); + String filterName = options.get("filterName"); + if (StringUtils.isNotEmpty(filterName)) { + String[] split = filterName.split("\\."); + if (split.length == 2) { + sqlWhere.append(" AND (TABLE_NAME LIKE '") + .append( + split[1].contains("%") + ? split[1].toUpperCase(Locale.ROOT) + : "%" + split[1].toUpperCase(Locale.ROOT) + "%") + .append("'") + .append(" AND OWNER LIKE '") + .append( + split[0].contains("%") + ? split[0].toUpperCase(Locale.ROOT) + : "%" + split[0].toUpperCase(Locale.ROOT) + "%") + .append("')"); + } else { + String filterNameRep = + filterName.contains("%") + ? filterName.toUpperCase(Locale.ROOT) + : "%" + filterName.toUpperCase(Locale.ROOT) + "%"; + sqlWhere.append(" AND (TABLE_NAME LIKE '%") + .append(filterNameRep) + .append("%'") + .append(" OR OWNER LIKE '%") + .append(filterNameRep) + .append("%')"); + } + } + sqlWhere.append(" ORDER BY OWNER, TABLE_NAME ) "); + String size = options.get("size"); + if (StringUtils.isNotEmpty(size)) { + sqlWhere.append("WHERE ROWNUM <= ").append(size); + } + log.info("execute sql :{}", sqlWhere.toString()); List tableNames = new ArrayList<>(); - try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[] {"TABLE"}); ) { - while (resultSet.next()) { - String tableName = resultSet.getString("TABLE_NAME"); - if (StringUtils.isNotBlank(tableName)) { - tableNames.add(tableName); + long start = System.currentTimeMillis(); + try (Connection connection = getConnection(requestParams); ) { + long end = System.currentTimeMillis(); + log.info("connection, cost {}ms for oracle", end - start); + start = System.currentTimeMillis(); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sqlWhere.toString())) { + end = System.currentTimeMillis(); + log.info("statement execute sql, cost {}ms for oracle", end - start); + start = System.currentTimeMillis(); + while (resultSet.next()) { + String schemaName = resultSet.getString("OWNER"); + String tableName = resultSet.getString("TABLE_NAME"); + tableNames.add(schemaName + "." + tableName); } + end = System.currentTimeMillis(); + log.info("while result set, cost {}ms for oracle", end - start); } return tableNames; } catch (ClassNotFoundException | SQLException e) { @@ -80,13 +138,12 @@ public List getDatabases( @NonNull String pluginName, @NonNull Map requestParams) { List dbNames = new ArrayList<>(); try (Connection connection = getConnection(requestParams); - PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + PreparedStatement statement = + connection.prepareStatement("SELECT NAME FROM v$database"); ResultSet re = statement.executeQuery()) { - // filter system databases while (re.next()) { - String dbName = re.getString("database"); - if (StringUtils.isNotBlank(dbName) - && !OracleDataSourceConfig.ORACLE_SYSTEM_DATABASES.contains(dbName)) { + String dbName = re.getString("NAME"); + if (StringUtils.isNotBlank(dbName)) { dbNames.add(dbName); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml index 9b07fb21a..27471250a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/pom.xml @@ -58,4 +58,23 @@ provided + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java index 9b2e9abfa..b21bf9930 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java @@ -26,6 +26,7 @@ import org.apache.commons.lang3.StringUtils; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -40,6 +41,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +@Slf4j public class PostgresqlDataSourceChannel implements DataSourceChannel { @Override @@ -57,18 +59,52 @@ public List getTables( @NonNull String pluginName, Map requestParams, String database, - Map option) { + Map options) { List tableNames = new ArrayList<>(); - String query = "SELECT table_schema, table_name FROM information_schema.tables"; - try (Connection connection = getConnection(requestParams, database)) { + StringBuilder queryWhere = new StringBuilder(); + String query = + "SELECT table_schema, table_name FROM information_schema.tables\n" + + "WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'root', 'pg_toast', 'pg_temp_1', 'pg_toast_temp_1', 'postgres', 'template0', 'template1')\n"; + queryWhere.append(query); + String filterName = options.get("filterName"); + if (StringUtils.isNotEmpty(filterName)) { + String[] split = filterName.split("\\."); + if (split.length == 2) { + queryWhere + .append("AND (table_schema LIKE '") + .append(split[0].contains("%") ? split[0] : "%" + split[0] + "%") + .append("'") + .append(" AND table_name LIKE '") + .append(split[1].contains("%") ? split[1] : "%" + split[1] + "%") + .append("')"); + } else { + String filterNameRep = + filterName.contains("%") ? filterName : "%" + filterName + "%"; + queryWhere + .append(" AND (table_schema LIKE '") + .append(filterNameRep) + .append("'") + .append(" OR table_name LIKE '") + .append(filterNameRep) + .append("')"); + } + } + String size = options.get("size"); + if (StringUtils.isNotEmpty(size)) { + queryWhere.append(" LIMIT ").append(size); + } + log.info(queryWhere.toString()); + requestParams.put( + PostgresqlOptionRule.URL.key(), + JdbcUtils.replaceDatabase( + requestParams.get(PostgresqlOptionRule.URL.key()), database)); + try (Connection connection = getConnection(requestParams)) { try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query)) { + ResultSet resultSet = statement.executeQuery(queryWhere.toString())) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); String tableName = resultSet.getString("table_name"); - if (StringUtils.isNotBlank(schemaName) - && !PostgresqlDataSourceConfig.POSTGRESQL_SYSTEM_DATABASES.contains( - schemaName)) { + if (StringUtils.isNotBlank(schemaName)) { tableNames.add(schemaName + "." + tableName); } } @@ -118,7 +154,11 @@ public List getTableFields( @NonNull String database, @NonNull String table) { List tableFields = new ArrayList<>(); - try (Connection connection = getConnection(requestParams, database); ) { + requestParams.put( + PostgresqlOptionRule.URL.key(), + JdbcUtils.replaceDatabase( + requestParams.get(PostgresqlOptionRule.URL.key()), database)); + try (Connection connection = getConnection(requestParams)) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, database, table); String[] split = table.split("\\."); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml index 440e4a451..7f48dca45 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/pom.xml @@ -59,4 +59,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml index 1ff5b6bca..6ce0dd22c 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/pom.xml @@ -59,4 +59,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java index cd3900912..e560e7f1f 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java @@ -24,6 +24,7 @@ import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -34,6 +35,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -57,18 +59,66 @@ public List getTables( @NonNull String pluginName, Map requestParams, String database, - Map option) { + Map options) { List tableNames = new ArrayList<>(); - try (Connection connection = getConnection(requestParams); - ResultSet resultSet = - connection - .getMetaData() - .getTables(database, null, null, new String[] {"TABLE"})) { - while (resultSet.next()) { - String tableName = resultSet.getString("TABLE_NAME"); - if (StringUtils.isNotBlank(tableName)) { - tableNames.add(tableName); + StringBuilder queryWhere = new StringBuilder(); + String query = + String.format( + "SELECT SCHEMA_NAME(schema_id) AS schema_name, name AS table_name\n" + + "FROM %s.sys.tables \n" + + "WHERE type = 'U' AND is_ms_shipped = 0 \n", + database); + queryWhere.append(query); + String filterName = options.get("filterName"); + if (StringUtils.isNotEmpty(filterName)) { + String[] split = filterName.split("\\."); + if (split.length == 2) { + String formatStr = + " AND (name LIKE '" + + (split[1].contains("%") ? split[1] : "%" + split[1] + "%") + + "' AND SCHEMA_NAME(schema_id) LIKE '" + + (split[0].contains("%") ? split[0] : "%" + split[0] + "%") + + "')"; + queryWhere.append(formatStr); + } else { + String filterNameRep = + filterName.contains("%") ? filterName : "%" + filterName + "%"; + String formatStr = + " AND (name LIKE '" + + filterNameRep + + "' OR SCHEMA_NAME(schema_id) LIKE '" + + filterNameRep + + "')"; + queryWhere.append(formatStr); + } + } + queryWhere.append("ORDER BY schema_name, table_name"); + String size = options.get("size"); + if (StringUtils.isNotEmpty(size)) { + queryWhere.append(" OFFSET 0 ROWS FETCH NEXT ").append(size).append(" ROWS ONLY"); + } + log.info("execute sql :{}", queryWhere.toString()); + long start = System.currentTimeMillis(); + try (Connection connection = getConnection(requestParams)) { + long end = System.currentTimeMillis(); + log.info("connection, cost {}ms for sqlserver", end - start); + start = System.currentTimeMillis(); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(queryWhere.toString())) { + end = System.currentTimeMillis(); + log.info("execute sql, cost {}ms for sqlserver", end - start); + start = System.currentTimeMillis(); + while (resultSet.next()) { + String schemaName = resultSet.getString("SCHEMA_NAME"); + String tableName = resultSet.getString("table_name"); + if (StringUtils.isNotBlank(schemaName) + && !SqlServerDataSourceConfig.SQLSERVER_SYSTEM_DATABASES.contains( + schemaName)) { + tableNames.add(schemaName + "." + tableName); + } } + end = System.currentTimeMillis(); + log.info("while result set, cost {}ms for sqlserver", end - start); } return tableNames; } catch (ClassNotFoundException | SQLException e) { @@ -115,11 +165,17 @@ public List getTableFields( @NonNull Map requestParams, @NonNull String database, @NonNull String table) { + Pair pair = parseSchemaAndTable(table); + return getTableFields(requestParams, database, pair.getLeft(), pair.getRight()); + } + + private List getTableFields( + Map requestParams, String dbName, String schemaName, String tableName) { List tableFields = new ArrayList<>(); - try (Connection connection = getConnection(requestParams, null)) { + try (Connection connection = getConnection(requestParams); ) { DatabaseMetaData metaData = connection.getMetaData(); - String primaryKey = getPrimaryKey(metaData, database, table); - try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) { + String primaryKey = getPrimaryKey(metaData, dbName, schemaName, tableName); + try (ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null)) { while (resultSet.next()) { TableField tableField = new TableField(); String columnName = resultSet.getString("COLUMN_NAME"); @@ -128,19 +184,33 @@ public List getTableFields( tableField.setPrimaryKey(true); } tableField.setName(columnName); - tableField.setType(resultSet.getString("TYPE_NAME")); + String typeString = resultSet.getString("TYPE_NAME"); + String[] parts = typeString.split(" "); + String baseType = parts.length > 0 ? parts[0] : ""; + tableField.setType(baseType); tableField.setComment(resultSet.getString("REMARKS")); Object nullable = resultSet.getObject("IS_NULLABLE"); - tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString())); + boolean isNullable = convertToBoolean(nullable); + tableField.setNullable(isNullable); tableFields.add(tableField); } } - } catch (ClassNotFoundException | SQLException e) { + } catch (Exception e) { throw new DataSourcePluginException("get table fields failed", e); } return tableFields; } + private boolean convertToBoolean(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } + if (value instanceof String) { + return value.equals("TRUE"); + } + return false; + } + @Override public Map> getTableFields( @NonNull String pluginName, @@ -150,11 +220,13 @@ public Map> getTableFields( return null; } - private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) + private String getPrimaryKey( + DatabaseMetaData metaData, String dbName, String schemaName, String tableName) throws SQLException { - ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); - while (primaryKeysInfo.next()) { - return primaryKeysInfo.getString("COLUMN_NAME"); + try (ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName)) { + while (primaryKeysInfo.next()) { + return primaryKeysInfo.getString("COLUMN_NAME"); + } } return null; } @@ -178,4 +250,12 @@ private Connection getConnection(Map requestParams, String datab } return DriverManager.getConnection(url); } + + private Pair parseSchemaAndTable(String tableName) { + String[] schemaAndTable = tableName.split("\\."); + if (schemaAndTable.length != 2) { + throw new DataSourcePluginException("table name is invalid"); + } + return Pair.of(schemaAndTable[0], schemaAndTable[1]); + } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java index 271653388..b8b8f5e5a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import static com.google.common.base.Preconditions.checkNotNull; @@ -169,11 +170,15 @@ private Connection getConnection(Map requestParams, String datab String url = JdbcUtils.replaceDatabase( requestParams.get(StarRocksOptionRule.URL.key()), databaseName); + + Properties info = new java.util.Properties(); + info.put("autoDeserialize", "false"); + info.put("allowLoadLocalInfile", "false"); + info.put("allowLoadLocalInfileInPath", ""); if (requestParams.containsKey(StarRocksOptionRule.USER.key())) { - String username = requestParams.get(StarRocksOptionRule.USER.key()); - String password = requestParams.get(StarRocksOptionRule.PASSWORD.key()); - return DriverManager.getConnection(url, username, password); + info.put("user", requestParams.get(StarRocksOptionRule.USER.key())); + info.put("password", requestParams.get(StarRocksOptionRule.PASSWORD.key())); } - return DriverManager.getConnection(url); + return DriverManager.getConnection(url, info); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml index 559a29361..477fad2ef 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/pom.xml @@ -58,4 +58,23 @@ provided + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java index 90f929370..59a09ee97 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.function.Function; import java.util.stream.Collectors; @@ -176,11 +177,14 @@ private Connection getConnection(Map requestParams, String datab String url = JdbcUtils.replaceDatabase( requestParams.get(TidbOptionRule.URL.key()), databaseName); + Properties info = new java.util.Properties(); + info.put("autoDeserialize", "false"); + info.put("allowLoadLocalInfile", "false"); + info.put("allowLoadLocalInfileInPath", ""); if (requestParams.containsKey(TidbOptionRule.USER.key())) { - String username = requestParams.get(TidbOptionRule.USER.key()); - String password = requestParams.get(TidbOptionRule.PASSWORD.key()); - return DriverManager.getConnection(url, username, password); + info.put("user", requestParams.get(TidbOptionRule.USER.key())); + info.put("password", requestParams.get(TidbOptionRule.PASSWORD.key())); } - return DriverManager.getConnection(url); + return DriverManager.getConnection(url, info); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml index ae68a64c5..8980db55d 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/pom.xml @@ -25,7 +25,7 @@ datasource-kafka - 3.2.0 + 3.4.0 @@ -61,4 +61,23 @@ ${kafka.client.version} + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml similarity index 58% rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml index 351f1cb6b..8bbf4dcf7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml @@ -22,7 +22,7 @@ ${revision} - datasource-s3redshift + datasource-mongodb @@ -31,38 +31,34 @@ ${project.version} provided + - org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber - 2.3.2 - - - org.apache.avro - avro - - - - - com.amazon.redshift - redshift-jdbc42 - ${redshift.version} - - - org.apache.hadoop - hadoop-aws - ${hadoop-aws.version} - - - jdk.tools - jdk.tools - - + org.mongodb + mongodb-driver-sync + 4.7.1 + - com.amazonaws - aws-java-sdk-bundle - ${aws-java-sdk-bundle.version} + org.apache.commons + commons-lang3 + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java new file mode 100644 index 000000000..55ba487d1 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apache.seatunnel.datasource.plugin.mongodb; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.collect.ImmutableList; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoIterable; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; + +@Slf4j +public class MongoDataSoueceChannel implements DataSourceChannel { + + private static final String DATABASE = "default"; + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return MongoOptionRule.optionRule(); + } + + @Override + public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + return MongoOptionRule.metadataRule(); + } + + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + + return Collections.emptyList(); + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + return ImmutableList.of(DATABASE); + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + return Collections.emptyList(); + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + + try (MongoClient mongoClient = createMongoClient(requestParams)) { + // Verify if the connection to mongodb was successful + MongoIterable databaseNames = mongoClient.listDatabaseNames(); + if (databaseNames.iterator().hasNext()) { + log.info("mongoDB connection successful"); + return true; + } else { + return false; + } + } catch (Exception e) { + throw new DataSourcePluginException( + "check MongoDB connectivity failed, " + e.getMessage(), e); + } + } + + // Resolve the URI in requestParams of Map type + private MongoClient createMongoClient(Map requestParams) { + + return MongoClients.create( + MongoRequestParamsUtils.parseStringFromRequestParams(requestParams)); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java similarity index 66% rename from seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java index ee6aa4ba0..b08f5b305 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.seatunnel.datasource.plugin.redshift.s3; +package com.apache.seatunnel.datasource.plugin.mongodb; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; @@ -28,29 +28,30 @@ import java.util.Set; @AutoService(DataSourceFactory.class) -public class S3RedshiftDataSourceFactory implements DataSourceFactory { +public class MongoDataSourceFactory implements DataSourceFactory { + public static final String MONGO_PLUGIN_NAME = "MongoDB"; + public static final String MONGO_PLUGIN_ICON = "MongoDB"; + public static final String MONGO_PLUGIN_VERSION = "1.0.0"; + @Override public String factoryIdentifier() { - return "S3-Redshift"; + return MONGO_PLUGIN_NAME; } @Override public Set supportedDataSources() { - DataSourcePluginInfo s3DatasourcePluginInfo = + return Sets.newHashSet( DataSourcePluginInfo.builder() - .name("S3-Redshift") - .type(DatasourcePluginTypeEnum.DATABASE.getCode()) - .version("1.0.0") - .supportVirtualTables(false) - .icon("S3-Redshift") - .icon("S3-Redshift") - .build(); - - return Sets.newHashSet(s3DatasourcePluginInfo); + .name(MONGO_PLUGIN_NAME) + .icon(MONGO_PLUGIN_ICON) + .version(MONGO_PLUGIN_VERSION) + .supportVirtualTables(true) + .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) + .build()); } @Override public DataSourceChannel createChannel() { - return new S3RedshiftDataSourceChannel(); + return new MongoDataSoueceChannel(); } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java new file mode 100644 index 000000000..cde0f89fd --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apache.seatunnel.datasource.plugin.mongodb; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.util.OptionRule; + +public class MongoOptionRule { + + public static final Option URI = + Options.key("uri") + .stringType() + .noDefaultValue() + .withDescription("The MongoDB connection uri."); + + public static final Option DATABASE = + Options.key("database") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB database to read or write."); + + public static final Option COLLECTION = + Options.key("collection") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB collection to read or write."); + + public static OptionRule optionRule() { + return OptionRule.builder().required(URI).build(); + } + + public static OptionRule metadataRule() { + return OptionRule.builder().required(DATABASE, COLLECTION).build(); + } +} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java similarity index 62% rename from seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java rename to seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java index 1dc971611..eb342acff 100644 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDescriptionDto.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Builder; -import lombok.Data; +package com.apache.seatunnel.datasource.plugin.mongodb; import java.util.Map; -@Data -@Builder -public class TaskDescriptionDto { - private String name; - private String executeScript; - private String content; - private Map params; - private int retryInterval; - private int retryTimes; +import static com.google.common.base.Preconditions.checkArgument; + +public class MongoRequestParamsUtils { + + public static String parseStringFromRequestParams(Map requestParams) { + checkArgument( + requestParams.containsKey(MongoOptionRule.URI.key()), + String.format("Missing %s in requestParams", MongoOptionRule.URI.key())); + + return requestParams.get(MongoOptionRule.URI.key()); + } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml index 23fedfd59..885b5bdf7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/pom.xml @@ -63,4 +63,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java index 4e5164fcf..e4a00fbdb 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; public class MysqlCDCDataSourceChannel implements DataSourceChannel { @@ -65,8 +66,8 @@ public List getTables( String pluginName, Map requestParams, String database, - Map option) { - return this.getTableNames(requestParams, database); + Map options) { + return this.getTableNames(requestParams, database, options); } @Override @@ -154,13 +155,17 @@ protected Connection init(Map requestParams) throws SQLException throw new DataSourcePluginException("Jdbc url is null"); } String url = requestParams.get(MysqlCDCOptionRule.BASE_URL.key()); + + Properties info = new java.util.Properties(); + info.put("autoDeserialize", "false"); + info.put("allowLoadLocalInfile", "false"); + info.put("allowLoadLocalInfileInPath", ""); if (null != requestParams.get(MysqlCDCOptionRule.PASSWORD.key()) && null != requestParams.get(MysqlCDCOptionRule.USERNAME.key())) { - String username = requestParams.get(MysqlCDCOptionRule.USERNAME.key()); - String password = requestParams.get(MysqlCDCOptionRule.PASSWORD.key()); - return DriverManager.getConnection(url, username, password); + info.put("user", requestParams.get(MysqlCDCOptionRule.USERNAME.key())); + info.put("password", requestParams.get(MysqlCDCOptionRule.PASSWORD.key())); } - return DriverManager.getConnection(url); + return DriverManager.getConnection(url, info); } protected List getDataBaseNames(Map requestParams) throws SQLException { @@ -179,17 +184,29 @@ protected List getDataBaseNames(Map requestParams) throw } } - protected List getTableNames(Map requestParams, String dbName) { + protected List getTableNames( + Map requestParams, String dbName, Map options) { List tableNames = new ArrayList<>(); + String filterName = options.get("filterName"); + String size = options.get("size"); + boolean isSize = StringUtils.isNotEmpty(size); + if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) { + filterName = "%" + filterName + "%"; + } else if (StringUtils.equals(filterName, "")) { + filterName = null; + } try (Connection connection = init(requestParams); ResultSet resultSet = connection .getMetaData() - .getTables(dbName, null, null, new String[] {"TABLE"})) { + .getTables(dbName, null, filterName, new String[] {"TABLE"})) { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); if (StringUtils.isNotBlank(tableName)) { tableNames.add(tableName); + if (isSize && tableNames.size() >= Integer.parseInt(size)) { + break; + } } } return tableNames; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java index 72a2ea208..8fb645d4a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java @@ -28,6 +28,8 @@ import java.sql.Connection; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public interface DataSourceChannel { @@ -71,11 +73,19 @@ List getTableFields( @NonNull String database, @NonNull String table); - Map> getTableFields( + default Map> getTableFields( @NonNull String pluginName, @NonNull Map requestParams, @NonNull String database, - @NonNull List tables); + @NonNull List tables) { + return tables.parallelStream() + .collect( + Collectors.toMap( + Function.identity(), + table -> + getTableFields( + pluginName, requestParams, database, table))); + } /** * just check metadata field is right and used by virtual table diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java deleted file mode 100644 index 8da509c12..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; - -import lombok.extern.slf4j.Slf4j; - -import java.util.Arrays; -import java.util.Map; - -import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; - -@Slf4j -public class HadoopS3AConfiguration { - - /* S3 constants */ - private static final String S3A_SCHEMA = "s3a"; - private static final String HDFS_S3N_IMPL = "org.apache.hadoop.fs.s3native.NativeS3FileSystem"; - private static final String HDFS_S3A_IMPL = "org.apache.hadoop.fs.s3a.S3AFileSystem"; - private static final String S3A_PROTOCOL = "s3a"; - private static final String DEFAULT_PROTOCOL = "s3n"; - private static final String S3_FORMAT_KEY = "fs.%s.%s"; - private static final String HDFS_IMPL_KEY = "impl"; - - public static Configuration getConfiguration(Map s3Options) { - - if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) { - throw new IllegalArgumentException( - "S3Redshift datasource bucket is null, please check your config"); - } - if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) { - throw new IllegalArgumentException( - "S3Redshift datasource endpoint is null, please check your config"); - } - String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key()); - - String protocol = DEFAULT_PROTOCOL; - if (bucket.startsWith(S3A_PROTOCOL)) { - protocol = S3A_PROTOCOL; - } - String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL; - Configuration hadoopConf = new Configuration(); - hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket); - hadoopConf.set( - S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(), - s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())); - hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl); - if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) { - Arrays.stream( - s3Options - .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key()) - .split("\n")) - .map(String::trim) - .filter(StringUtils::isNotBlank) - .forEach( - line -> { - String[] kv = line.split("="); - if (kv.length == 2) { - hadoopConf.set(kv[0].trim(), kv[1].trim()); - } - }); - } - if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider - .getProvider() - .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { - hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); - hadoopConf.set( - "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key())); - hadoopConf.set( - "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key())); - } else { - hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); - } - return hadoopConf; - } - - private static String formatKey(String protocol, String key) { - return String.format(S3_FORMAT_KEY, protocol, key); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java deleted file mode 100644 index de9e2b09e..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; -import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import com.google.common.collect.Sets; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Slf4j -public class S3RedshiftDataSourceChannel implements DataSourceChannel { - - @Override - public OptionRule getDataSourceOptions(@NonNull String pluginName) { - return S3RedshiftOptionRule.optionRule(); - } - - @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { - return S3RedshiftOptionRule.metadataRule(); - } - - @Override - public List getTables( - @NonNull String pluginName, Map requestParams, String database) { - return getTableNames(requestParams, database); - } - - @Override - public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { - try { - return getDataBaseNames(pluginName, requestParams); - } catch (SQLException e) { - throw new DataSourcePluginException("Query redshift databases failed", e); - } - } - - @Override - public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { - checkHdfsS3Connection(requestParams); - checkJdbcConnection(requestParams); - return true; - } - - @Override - public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { - return getTableFields(requestParams, database, table); - } - - @Override - public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { - // not need this method - return null; - } - - private void checkJdbcConnection(Map requestParams) { - String jdbcUrl = requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()); - String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key()); - String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()); - if (StringUtils.isBlank(jdbcUrl)) { - throw new DataSourcePluginException("Redshift Jdbc url is empty"); - } - if (StringUtils.isBlank(username) && StringUtils.isBlank(password)) { - try (Connection ignored = DriverManager.getConnection(jdbcUrl)) { - log.info("Redshift jdbc connection is valid"); - return; - } catch (SQLException e) { - throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); - } - } - try (Connection ignored = DriverManager.getConnection(jdbcUrl, username, password)) { - log.info("Redshift jdbc connection is valid"); - } catch (SQLException e) { - throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); - } - } - - private void checkHdfsS3Connection(Map requestParams) { - Configuration s3Conf = HadoopS3AConfiguration.getConfiguration(requestParams); - try (FileSystem fs = FileSystem.get(s3Conf)) { - fs.getFileStatus(new org.apache.hadoop.fs.Path("/")); - } catch (IOException e) { - throw new DataSourcePluginException( - "S3 configuration is invalid, please check your config", e); - } - } - - protected Connection init(Map requestParams, String databaseName) - throws SQLException { - if (null == requestParams.get(S3RedshiftOptionRule.JDBC_URL.key())) { - throw new DataSourcePluginException("Jdbc url is null"); - } - String url = - JdbcUtils.replaceDatabase( - requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName); - if (null != requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()) - && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) { - String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key()); - String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()); - return DriverManager.getConnection(url, username, password); - } - return DriverManager.getConnection(url); - } - - protected List getDataBaseNames(String pluginName, Map requestParams) - throws SQLException { - List dbNames = new ArrayList<>(); - try (Connection connection = init(requestParams, null); - PreparedStatement statement = - connection.prepareStatement("select datname from pg_database;"); - ResultSet re = statement.executeQuery()) { - while (re.next()) { - String dbName = re.getString("datname"); - if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) { - dbNames.add(dbName); - } - } - return dbNames; - } catch (SQLException e) { - throw new DataSourcePluginException("get databases failed", e); - } - } - - protected List getTableNames(Map requestParams, String dbName) { - List tableNames = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName); ) { - ResultSet resultSet = - connection.getMetaData().getTables(dbName, null, null, new String[] {"TABLE"}); - while (resultSet.next()) { - String tableName = resultSet.getString("TABLE_NAME"); - if (StringUtils.isNotBlank(tableName)) { - tableNames.add(tableName); - } - } - return tableNames; - } catch (SQLException e) { - throw new DataSourcePluginException("get table names failed", e); - } - } - - protected List getTableFields( - Map requestParams, String dbName, String tableName) { - List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName); ) { - DatabaseMetaData metaData = connection.getMetaData(); - String primaryKey = getPrimaryKey(metaData, dbName, tableName); - String[] split = tableName.split("\\."); - if (split.length != 2) { - throw new DataSourcePluginException( - "Postgresql tableName should composed by schemaName.tableName"); - } - ResultSet resultSet = metaData.getColumns(dbName, split[0], split[1], null); - while (resultSet.next()) { - TableField tableField = new TableField(); - String columnName = resultSet.getString("COLUMN_NAME"); - tableField.setPrimaryKey(false); - if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) { - tableField.setPrimaryKey(true); - } - tableField.setName(columnName); - tableField.setType(resultSet.getString("TYPE_NAME")); - tableField.setComment(resultSet.getString("REMARKS")); - Object nullable = resultSet.getObject("IS_NULLABLE"); - boolean isNullable = convertToBoolean(nullable); - tableField.setNullable(isNullable); - tableFields.add(tableField); - } - } catch (SQLException e) { - throw new DataSourcePluginException("get table fields failed", e); - } - return tableFields; - } - - private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { - ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); - while (primaryKeysInfo.next()) { - return primaryKeysInfo.getString("COLUMN_NAME"); - } - return null; - } - - @SuppressWarnings("checkstyle:MagicNumber") - private static boolean checkHostConnectable(String host, int port) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 1000); - return true; - } catch (IOException e) { - - throw new DataSourcePluginException("check host connectable failed", e); - } - } - - private boolean isNotSystemDatabase(String dbName) { - return !POSTGRESQL_SYSTEM_DATABASES.contains(dbName.toLowerCase()); - } - - private boolean convertToBoolean(Object value) { - if (value instanceof Boolean) { - return (Boolean) value; - } - if (value instanceof String) { - return value.equals("TRUE"); - } - return false; - } - - public static final Set POSTGRESQL_SYSTEM_DATABASES = - Sets.newHashSet( - "information_schema", - "pg_catalog", - "root", - "pg_toast", - "pg_temp_1", - "pg_toast_temp_1", - "postgres", - "template0", - "template1"); -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java deleted file mode 100644 index 5d69ee803..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; -import org.apache.seatunnel.api.configuration.util.OptionRule; - -import java.util.Arrays; -import java.util.Map; - -public class S3RedshiftOptionRule { - - public static final Option JDBC_URL = - Options.key("jdbc_url") - .stringType() - .noDefaultValue() - .withDescription( - "Redshift jdbc connection url, eg: jdbc:redshift://localhost:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); - - public static final Option JDBC_USER = - Options.key("jdbc_user") - .stringType() - .noDefaultValue() - .withDescription("Redshift jdbc connection user"); - - public static final Option JDBC_PASSWORD = - Options.key("jdbc_password") - .stringType() - .noDefaultValue() - .withDescription("Redshift jdbc connection password"); - - public static final Option ACCESS_KEY = - Options.key("access_key") - .stringType() - .noDefaultValue() - .withDescription("S3 access key"); - - public static final Option SECRET_KEY = - Options.key("secret_key") - .stringType() - .noDefaultValue() - .withDescription("S3 secret key"); - - public static final Option BUCKET = - Options.key("bucket").stringType().noDefaultValue().withDescription("S3 bucket name"); - - public static final Option FS_S3A_ENDPOINT = - Options.key("fs.s3a.endpoint") - .stringType() - .noDefaultValue() - .withDescription("fs s3a endpoint"); - - public static final Option S3A_AWS_CREDENTIALS_PROVIDER = - Options.key("fs.s3a.aws.credentials.provider") - .enumType(S3aAwsCredentialsProvider.class) - .defaultValue(S3aAwsCredentialsProvider.InstanceProfileCredentialsProvider) - .withDescription("s3a aws credentials provider"); - - public static final Option> HADOOP_S3_PROPERTIES = - Options.key("hadoop_s3_properties") - .mapType() - .noDefaultValue() - .withDescription( - "{" - + "fs.s3a.buffer.dir = /data/st_test/s3a\n" - + "fs.s3a.fast.upload.buffer = disk\n" - + "}"); - - public static OptionRule optionRule() { - return OptionRule.builder() - .required( - JDBC_URL, - BUCKET, - FS_S3A_ENDPOINT, - S3A_AWS_CREDENTIALS_PROVIDER, - JDBC_USER, - JDBC_PASSWORD) - .optional(HADOOP_S3_PROPERTIES) - .conditional( - S3A_AWS_CREDENTIALS_PROVIDER, - S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider, - ACCESS_KEY, - SECRET_KEY) - .build(); - } - - public static final Option PATH = - Options.key("path").stringType().noDefaultValue().withDescription("S3 write path"); - - public static final Option TYPE = - Options.key("file_format_type") - .enumType(FileFormat.class) - .noDefaultValue() - .withDescription("S3 write type"); - - public static final Option DELIMITER = - Options.key("delimiter") - .stringType() - .noDefaultValue() - .withDescription("S3 write delimiter"); - - public static final Option> SCHEMA = - Options.key("schema").mapType().noDefaultValue().withDescription("SeaTunnel Schema"); - - public static final Option PARSE_PARSE_PARTITION_FROM_PATH = - Options.key("parse_partition_from_path") - .booleanType() - .noDefaultValue() - .withDescription("S3 write parse_partition_from_path"); - - public static final Option DATE_FORMAT = - Options.key("date_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write date_format"); - - public static final Option DATETIME_FORMAT = - Options.key("time_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write time_format"); - - public static final Option TIME_FORMAT = - Options.key("datetime_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write datetime_format"); - - public static OptionRule metadataRule() { - return OptionRule.builder() - .required(PATH, TYPE) - .conditional(TYPE, FileFormat.TEXT, DELIMITER) - .conditional(TYPE, Arrays.asList(FileFormat.TEXT, FileFormat.JSON), SCHEMA) - .optional(PARSE_PARSE_PARTITION_FROM_PATH) - .optional(DATE_FORMAT) - .optional(DATETIME_FORMAT) - .optional(TIME_FORMAT) - .build(); - } - - public enum S3aAwsCredentialsProvider { - SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"), - - InstanceProfileCredentialsProvider("com.amazonaws.auth.InstanceProfileCredentialsProvider"); - - private String provider; - - S3aAwsCredentialsProvider(String provider) { - this.provider = provider; - } - - public String getProvider() { - return provider; - } - - @Override - public String toString() { - return provider; - } - } - - public enum FileFormat { - CSV("csv"), - TEXT("txt"), - PARQUET("parquet"), - ORC("orc"), - JSON("json"); - - private final String type; - - FileFormat(String type) { - this.type = type; - } - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml index ae402efd2..047fbb74f 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/pom.xml @@ -45,4 +45,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java deleted file mode 100644 index 8da509c12..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; - -import lombok.extern.slf4j.Slf4j; - -import java.util.Arrays; -import java.util.Map; - -import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; - -@Slf4j -public class HadoopS3AConfiguration { - - /* S3 constants */ - private static final String S3A_SCHEMA = "s3a"; - private static final String HDFS_S3N_IMPL = "org.apache.hadoop.fs.s3native.NativeS3FileSystem"; - private static final String HDFS_S3A_IMPL = "org.apache.hadoop.fs.s3a.S3AFileSystem"; - private static final String S3A_PROTOCOL = "s3a"; - private static final String DEFAULT_PROTOCOL = "s3n"; - private static final String S3_FORMAT_KEY = "fs.%s.%s"; - private static final String HDFS_IMPL_KEY = "impl"; - - public static Configuration getConfiguration(Map s3Options) { - - if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) { - throw new IllegalArgumentException( - "S3Redshift datasource bucket is null, please check your config"); - } - if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) { - throw new IllegalArgumentException( - "S3Redshift datasource endpoint is null, please check your config"); - } - String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key()); - - String protocol = DEFAULT_PROTOCOL; - if (bucket.startsWith(S3A_PROTOCOL)) { - protocol = S3A_PROTOCOL; - } - String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL; - Configuration hadoopConf = new Configuration(); - hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket); - hadoopConf.set( - S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(), - s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())); - hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl); - if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) { - Arrays.stream( - s3Options - .get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key()) - .split("\n")) - .map(String::trim) - .filter(StringUtils::isNotBlank) - .forEach( - line -> { - String[] kv = line.split("="); - if (kv.length == 2) { - hadoopConf.set(kv[0].trim(), kv[1].trim()); - } - }); - } - if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider - .getProvider() - .equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) { - hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); - hadoopConf.set( - "fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key())); - hadoopConf.set( - "fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key())); - } else { - hadoopConf.set( - S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(), - s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key())); - } - return hadoopConf; - } - - private static String formatKey(String protocol, String key) { - return String.format(S3_FORMAT_KEY, protocol, key); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java deleted file mode 100644 index 47660dc4c..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftDataSourceChannel.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; -import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import com.google.common.collect.Sets; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Slf4j -public class S3RedshiftDataSourceChannel implements DataSourceChannel { - - @Override - public OptionRule getDataSourceOptions(@NonNull String pluginName) { - return S3RedshiftOptionRule.optionRule(); - } - - @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { - return S3RedshiftOptionRule.metadataRule(); - } - - @Override - public List getTables( - @NonNull String pluginName, - Map requestParams, - String database, - Map option) { - return getTableNames(requestParams, database); - } - - @Override - public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { - try { - return getDataBaseNames(pluginName, requestParams); - } catch (SQLException e) { - throw new DataSourcePluginException("Query redshift databases failed", e); - } - } - - @Override - public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { - checkHdfsS3Connection(requestParams); - checkJdbcConnection(requestParams); - return true; - } - - @Override - public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { - return getTableFields(requestParams, database, table); - } - - @Override - public Map> getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull List tables) { - // not need this method - return null; - } - - private void checkJdbcConnection(Map requestParams) { - String jdbcUrl = requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()); - String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key()); - String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()); - if (StringUtils.isBlank(jdbcUrl)) { - throw new DataSourcePluginException("Redshift Jdbc url is empty"); - } - if (StringUtils.isBlank(username) && StringUtils.isBlank(password)) { - try (Connection ignored = DriverManager.getConnection(jdbcUrl)) { - log.info("Redshift jdbc connection is valid"); - return; - } catch (SQLException e) { - throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); - } - } - try (Connection ignored = DriverManager.getConnection(jdbcUrl, username, password)) { - log.info("Redshift jdbc connection is valid"); - } catch (SQLException e) { - throw new DataSourcePluginException( - "Check Redshift jdbc connection failed,please check your config", e); - } - } - - private void checkHdfsS3Connection(Map requestParams) { - Configuration s3Conf = HadoopS3AConfiguration.getConfiguration(requestParams); - try (FileSystem fs = FileSystem.get(s3Conf)) { - fs.getFileStatus(new org.apache.hadoop.fs.Path("/")); - } catch (IOException e) { - throw new DataSourcePluginException( - "S3 configuration is invalid, please check your config", e); - } - } - - protected Connection init(Map requestParams, String databaseName) - throws SQLException { - if (null == requestParams.get(S3RedshiftOptionRule.JDBC_URL.key())) { - throw new DataSourcePluginException("Jdbc url is null"); - } - String url = - JdbcUtils.replaceDatabase( - requestParams.get(S3RedshiftOptionRule.JDBC_URL.key()), databaseName); - if (null != requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()) - && null != requestParams.get(S3RedshiftOptionRule.JDBC_USER.key())) { - String username = requestParams.get(S3RedshiftOptionRule.JDBC_USER.key()); - String password = requestParams.get(S3RedshiftOptionRule.JDBC_PASSWORD.key()); - return DriverManager.getConnection(url, username, password); - } - return DriverManager.getConnection(url); - } - - protected List getDataBaseNames(String pluginName, Map requestParams) - throws SQLException { - List dbNames = new ArrayList<>(); - try (Connection connection = init(requestParams, null); - PreparedStatement statement = - connection.prepareStatement("select datname from pg_database;"); - ResultSet re = statement.executeQuery()) { - while (re.next()) { - String dbName = re.getString("datname"); - if (StringUtils.isNotBlank(dbName) && isNotSystemDatabase(dbName)) { - dbNames.add(dbName); - } - } - return dbNames; - } catch (SQLException e) { - throw new DataSourcePluginException("get databases failed", e); - } - } - - protected List getTableNames(Map requestParams, String dbName) { - List tableNames = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName); ) { - ResultSet resultSet = - connection.getMetaData().getTables(dbName, null, null, new String[] {"TABLE"}); - while (resultSet.next()) { - String tableName = resultSet.getString("TABLE_NAME"); - if (StringUtils.isNotBlank(tableName)) { - tableNames.add(tableName); - } - } - return tableNames; - } catch (SQLException e) { - throw new DataSourcePluginException("get table names failed", e); - } - } - - protected List getTableFields( - Map requestParams, String dbName, String tableName) { - List tableFields = new ArrayList<>(); - try (Connection connection = init(requestParams, dbName); ) { - DatabaseMetaData metaData = connection.getMetaData(); - String primaryKey = getPrimaryKey(metaData, dbName, tableName); - String[] split = tableName.split("\\."); - if (split.length != 2) { - throw new DataSourcePluginException( - "Postgresql tableName should composed by schemaName.tableName"); - } - ResultSet resultSet = metaData.getColumns(dbName, split[0], split[1], null); - while (resultSet.next()) { - TableField tableField = new TableField(); - String columnName = resultSet.getString("COLUMN_NAME"); - tableField.setPrimaryKey(false); - if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) { - tableField.setPrimaryKey(true); - } - tableField.setName(columnName); - tableField.setType(resultSet.getString("TYPE_NAME")); - tableField.setComment(resultSet.getString("REMARKS")); - Object nullable = resultSet.getObject("IS_NULLABLE"); - boolean isNullable = convertToBoolean(nullable); - tableField.setNullable(isNullable); - tableFields.add(tableField); - } - } catch (SQLException e) { - throw new DataSourcePluginException("get table fields failed", e); - } - return tableFields; - } - - private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) - throws SQLException { - ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName); - while (primaryKeysInfo.next()) { - return primaryKeysInfo.getString("COLUMN_NAME"); - } - return null; - } - - @SuppressWarnings("checkstyle:MagicNumber") - private static boolean checkHostConnectable(String host, int port) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 1000); - return true; - } catch (IOException e) { - - throw new DataSourcePluginException("check host connectable failed", e); - } - } - - private boolean isNotSystemDatabase(String dbName) { - return !POSTGRESQL_SYSTEM_DATABASES.contains(dbName.toLowerCase()); - } - - private boolean convertToBoolean(Object value) { - if (value instanceof Boolean) { - return (Boolean) value; - } - if (value instanceof String) { - return value.equals("TRUE"); - } - return false; - } - - public static final Set POSTGRESQL_SYSTEM_DATABASES = - Sets.newHashSet( - "information_schema", - "pg_catalog", - "root", - "pg_toast", - "pg_temp_1", - "pg_toast_temp_1", - "postgres", - "template0", - "template1"); -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java deleted file mode 100644 index 5d69ee803..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/S3RedshiftOptionRule.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.datasource.plugin.redshift.s3; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; -import org.apache.seatunnel.api.configuration.util.OptionRule; - -import java.util.Arrays; -import java.util.Map; - -public class S3RedshiftOptionRule { - - public static final Option JDBC_URL = - Options.key("jdbc_url") - .stringType() - .noDefaultValue() - .withDescription( - "Redshift jdbc connection url, eg: jdbc:redshift://localhost:5439/test?useSSL=false&serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8"); - - public static final Option JDBC_USER = - Options.key("jdbc_user") - .stringType() - .noDefaultValue() - .withDescription("Redshift jdbc connection user"); - - public static final Option JDBC_PASSWORD = - Options.key("jdbc_password") - .stringType() - .noDefaultValue() - .withDescription("Redshift jdbc connection password"); - - public static final Option ACCESS_KEY = - Options.key("access_key") - .stringType() - .noDefaultValue() - .withDescription("S3 access key"); - - public static final Option SECRET_KEY = - Options.key("secret_key") - .stringType() - .noDefaultValue() - .withDescription("S3 secret key"); - - public static final Option BUCKET = - Options.key("bucket").stringType().noDefaultValue().withDescription("S3 bucket name"); - - public static final Option FS_S3A_ENDPOINT = - Options.key("fs.s3a.endpoint") - .stringType() - .noDefaultValue() - .withDescription("fs s3a endpoint"); - - public static final Option S3A_AWS_CREDENTIALS_PROVIDER = - Options.key("fs.s3a.aws.credentials.provider") - .enumType(S3aAwsCredentialsProvider.class) - .defaultValue(S3aAwsCredentialsProvider.InstanceProfileCredentialsProvider) - .withDescription("s3a aws credentials provider"); - - public static final Option> HADOOP_S3_PROPERTIES = - Options.key("hadoop_s3_properties") - .mapType() - .noDefaultValue() - .withDescription( - "{" - + "fs.s3a.buffer.dir = /data/st_test/s3a\n" - + "fs.s3a.fast.upload.buffer = disk\n" - + "}"); - - public static OptionRule optionRule() { - return OptionRule.builder() - .required( - JDBC_URL, - BUCKET, - FS_S3A_ENDPOINT, - S3A_AWS_CREDENTIALS_PROVIDER, - JDBC_USER, - JDBC_PASSWORD) - .optional(HADOOP_S3_PROPERTIES) - .conditional( - S3A_AWS_CREDENTIALS_PROVIDER, - S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider, - ACCESS_KEY, - SECRET_KEY) - .build(); - } - - public static final Option PATH = - Options.key("path").stringType().noDefaultValue().withDescription("S3 write path"); - - public static final Option TYPE = - Options.key("file_format_type") - .enumType(FileFormat.class) - .noDefaultValue() - .withDescription("S3 write type"); - - public static final Option DELIMITER = - Options.key("delimiter") - .stringType() - .noDefaultValue() - .withDescription("S3 write delimiter"); - - public static final Option> SCHEMA = - Options.key("schema").mapType().noDefaultValue().withDescription("SeaTunnel Schema"); - - public static final Option PARSE_PARSE_PARTITION_FROM_PATH = - Options.key("parse_partition_from_path") - .booleanType() - .noDefaultValue() - .withDescription("S3 write parse_partition_from_path"); - - public static final Option DATE_FORMAT = - Options.key("date_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write date_format"); - - public static final Option DATETIME_FORMAT = - Options.key("time_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write time_format"); - - public static final Option TIME_FORMAT = - Options.key("datetime_format") - .stringType() - .noDefaultValue() - .withDescription("S3 write datetime_format"); - - public static OptionRule metadataRule() { - return OptionRule.builder() - .required(PATH, TYPE) - .conditional(TYPE, FileFormat.TEXT, DELIMITER) - .conditional(TYPE, Arrays.asList(FileFormat.TEXT, FileFormat.JSON), SCHEMA) - .optional(PARSE_PARSE_PARTITION_FROM_PATH) - .optional(DATE_FORMAT) - .optional(DATETIME_FORMAT) - .optional(TIME_FORMAT) - .build(); - } - - public enum S3aAwsCredentialsProvider { - SimpleAWSCredentialsProvider("org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"), - - InstanceProfileCredentialsProvider("com.amazonaws.auth.InstanceProfileCredentialsProvider"); - - private String provider; - - S3aAwsCredentialsProvider(String provider) { - this.provider = provider; - } - - public String getProvider() { - return provider; - } - - @Override - public String toString() { - return provider; - } - } - - public enum FileFormat { - CSV("csv"), - TEXT("txt"), - PARQUET("parquet"), - ORC("orc"), - JSON("json"); - - private final String type; - - FileFormat(String type) { - this.type = type; - } - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml index 2f9fe5881..45f724e86 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/pom.xml @@ -52,4 +52,23 @@ provided + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java index a3d02336f..fb47afd2b 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java @@ -69,7 +69,7 @@ public List getTables( Map requestParams, String database, Map options) { - return this.getTableNames(requestParams, database); + return this.getTableNames(requestParams, database, options); } @Override @@ -146,20 +146,52 @@ private List getDataBaseNames(Map requestParams) throws } } - private List getTableNames(Map requestParams, String dbName) { + private List getTableNames( + Map requestParams, String dbName, Map options) { + StringBuilder sqlWhere = new StringBuilder(); final String sql = String.format( "SELECT SCHEMAS.NAME AS SCHEMA_NAME, TABLES.NAME AS TABLE_NAME" - + " FROM %s.SYS.SCHEMAS AS SCHEMAS" - + " JOIN %s.SYS.TABLES AS TABLES" + + " FROM %s.sys.schemas AS SCHEMAS" + + " JOIN %s.sys.tables AS TABLES" + " ON SCHEMAS.SCHEMA_ID = TABLES.SCHEMA_ID" + " AND TABLES.IS_TRACKED_BY_CDC = 1", dbName, dbName); + sqlWhere.append(sql); + String filterName = options.get("filterName"); + if (StringUtils.isNotEmpty(filterName)) { + String[] split = filterName.split("\\."); + if (split.length == 2) { + String formatStr = + " AND (TABLES.NAME LIKE '" + + (split[1].contains("%") ? split[1] : "%" + split[1] + "%") + + "' AND SCHEMAS.NAME LIKE '" + + (split[0].contains("%") ? split[0] : "%" + split[0] + "%") + + "')"; + sqlWhere.append(formatStr); + } else { + String filterNameRep = + filterName.contains("%") ? filterName : "%" + filterName + "%"; + String formatStr = + " AND (TABLES.NAME LIKE '" + + filterNameRep + + "' OR SCHEMAS.NAME LIKE '" + + filterNameRep + + "')"; + sqlWhere.append(formatStr); + } + } + sqlWhere.append(" ORDER BY SCHEMAS.NAME, TABLES.NAME"); + String size = options.get("size"); + if (StringUtils.isNotEmpty(size)) { + sqlWhere.append(" OFFSET 0 ROWS FETCH NEXT ").append(size).append(" ROWS ONLY"); + } + log.info("execute sql :{}", sqlWhere.toString()); List tableNames = new ArrayList<>(); try (Connection connection = init(requestParams); Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sql)) { + ResultSet resultSet = statement.executeQuery(sqlWhere.toString())) { while (resultSet.next()) { String schemaName = resultSet.getString("SCHEMA_NAME"); String tableName = resultSet.getString("TABLE_NAME"); @@ -177,21 +209,25 @@ private List getTableFields( try (Connection connection = init(requestParams); ) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, dbName, schemaName, tableName); - ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null); - while (resultSet.next()) { - TableField tableField = new TableField(); - String columnName = resultSet.getString("COLUMN_NAME"); - tableField.setPrimaryKey(false); - if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) { - tableField.setPrimaryKey(true); + try (ResultSet resultSet = metaData.getColumns(dbName, schemaName, tableName, null)) { + while (resultSet.next()) { + TableField tableField = new TableField(); + String columnName = resultSet.getString("COLUMN_NAME"); + tableField.setPrimaryKey(false); + if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) { + tableField.setPrimaryKey(true); + } + tableField.setName(columnName); + String typeString = resultSet.getString("TYPE_NAME"); + String[] parts = typeString.split(" "); + String baseType = parts.length > 0 ? parts[0] : ""; + tableField.setType(baseType); + tableField.setComment(resultSet.getString("REMARKS")); + Object nullable = resultSet.getObject("IS_NULLABLE"); + boolean isNullable = convertToBoolean(nullable); + tableField.setNullable(isNullable); + tableFields.add(tableField); } - tableField.setName(columnName); - tableField.setType(resultSet.getString("TYPE_NAME")); - tableField.setComment(resultSet.getString("REMARKS")); - Object nullable = resultSet.getObject("IS_NULLABLE"); - boolean isNullable = convertToBoolean(nullable); - tableField.setNullable(isNullable); - tableFields.add(tableField); } } catch (SQLException e) { throw new DataSourcePluginException("get table fields failed", e); @@ -202,9 +238,10 @@ private List getTableFields( private String getPrimaryKey( DatabaseMetaData metaData, String dbName, String schemaName, String tableName) throws SQLException { - ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName); - while (primaryKeysInfo.next()) { - return primaryKeysInfo.getString("COLUMN_NAME"); + try (ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, schemaName, tableName)) { + while (primaryKeysInfo.next()) { + return primaryKeysInfo.getString("COLUMN_NAME"); + } } return null; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/pom.xml index 07e1d40c8..14ac1a851 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/pom.xml @@ -57,4 +57,22 @@ + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java index 4acc43e60..14a1e1b40 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksCatalog.java @@ -39,6 +39,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Properties; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -79,7 +80,7 @@ public StarRocksCatalog(String catalogName, String username, String pwd, String public List listDatabases() throws CatalogException { List databases = new ArrayList<>(); - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); + try (Connection conn = getConnection(defaultUrl); PreparedStatement ps = conn.prepareStatement("SHOW DATABASES;"); ResultSet rs = ps.executeQuery(); ) { @@ -103,7 +104,7 @@ public List listTables(String databaseName) throw new DatabaseNotExistException(this.catalogName, databaseName); } - try (Connection conn = DriverManager.getConnection(baseUrl + databaseName, username, pwd); + try (Connection conn = getConnection(baseUrl + databaseName); PreparedStatement ps = conn.prepareStatement("SHOW TABLES;"); ResultSet rs = ps.executeQuery()) { @@ -127,7 +128,7 @@ public List getTable(TablePath tablePath) } String dbUrl = baseUrl + tablePath.getDatabaseName(); - try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd); + try (Connection conn = getConnection(dbUrl); PreparedStatement statement = conn.prepareStatement( String.format( @@ -178,7 +179,7 @@ public static String splitDefaultUrl(String defaultUrl) { protected Optional getPrimaryKey(String schema, String table) throws SQLException { List pkFields = new ArrayList<>(); - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); + try (Connection conn = getConnection(defaultUrl); PreparedStatement statement = conn.prepareStatement( String.format( @@ -222,4 +223,14 @@ public boolean tableExists(TablePath tablePath) throws CatalogException { return false; } } + + protected Connection getConnection(String url) throws SQLException { + Properties info = new java.util.Properties(); + info.put("autoDeserialize", "false"); + info.put("allowLoadLocalInfile", "false"); + info.put("allowLoadLocalInfileInPath", ""); + info.put("user", username); + info.put("password", pwd); + return DriverManager.getConnection(url, info); + } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml index cc10fdb26..13af3009f 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml @@ -31,7 +31,7 @@ datasource-kafka datasource-elasticsearch datasource-jdbc-hive - datasource-s3redshift + datasource-hive datasource-starrocks datasource-jdbc-clickhouse datasource-jdbc-mysql @@ -44,24 +44,8 @@ datasource-s3 datasource-sqlserver-cdc datasource-jdbc-tidb + datasource-mongodb + datasource-jdbc-db2 - - - - org.apache.maven.plugins - maven-dependency-plugin - - ${e2e.dependency.skip} - true - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - diff --git a/seatunnel-server/pom.xml b/seatunnel-server/pom.xml index 0a623719c..fd3454344 100644 --- a/seatunnel-server/pom.xml +++ b/seatunnel-server/pom.xml @@ -27,8 +27,6 @@ seatunnel-app seatunnel-dynamicform - seatunnel-spi - seatunnel-scheduler seatunnel-server-common diff --git a/seatunnel-server/seatunnel-app/pom.xml b/seatunnel-server/seatunnel-app/pom.xml index 1d481a5e1..24c2a44fe 100644 --- a/seatunnel-server/seatunnel-app/pom.xml +++ b/seatunnel-server/seatunnel-app/pom.xml @@ -28,34 +28,26 @@ - io.swagger.core.v3 + io.swagger swagger-annotations - 2.2.14 org.awaitility awaitility - 4.2.0 + test + + + io.swagger.core.v3 + swagger-annotations + 2.2.14 org.apache.seatunnel seatunnel-common - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel seatunnel-api - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel @@ -71,12 +63,6 @@ org.apache.seatunnel seatunnel-plugin-discovery - - - org.apache.logging.log4j - log4j-slf4j-impl - - @@ -115,13 +101,6 @@ - - - cn.hutool - hutool-all - 5.8.16 - - org.apache.seatunnel @@ -139,404 +118,156 @@ org.apache.seatunnel connector-common - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - - - org.apache.seatunnel - seatunnel-transforms-v2 - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-console - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.seatunnel connector-fake - provided - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-kafka - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-base - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-myhours - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-lemlist - provided - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-klaviyo - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-onesignal - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-notion - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-jdbc - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-cdc-mysql - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.apache.seatunnel - connector-cdc-sqlserver - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-clickhouse - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-hive - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-hadoop - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-local - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-oss - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-ftp - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-sftp - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-hudi - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-email - provided - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-elasticsearch - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-neo4j - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-redis - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-sentry - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-iceberg - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-influxdb - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-cassandra - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-file-s3 - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-tablestore - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-slack - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-http-jira - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-rabbitmq - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-openmldb - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-doris - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-maxcompute - - - org.apache.logging.log4j - log4j-slf4j-impl - - org.apache.seatunnel connector-cdc-sqlserver - - - org.apache.logging.log4j - log4j-slf4j-impl - - @@ -557,7 +288,6 @@ org.springframework.boot spring-boot-starter-logging - @@ -620,11 +350,6 @@ springfox-swagger-ui - - io.swagger - swagger-annotations - - org.hibernate.validator hibernate-validator @@ -640,30 +365,11 @@ org.apache.commons commons-lang3 - 3.12.0 com.h2database h2 - ${h2.version} - - - - org.apache.seatunnel - seatunnel-spi - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.seatunnel - seatunnel-scheduler-dolphinscheduler - ${project.version} @@ -694,11 +400,6 @@ commons-io - - - com.cronutils - cron-utils - com.google.auto.service auto-service-annotations @@ -710,6 +411,7 @@ org.apache.seatunnel datasource-s3 + ${project.version} provided @@ -722,40 +424,34 @@ org.apache.seatunnel datasource-kafka - 1.0.0-SNAPSHOT - provided + ${project.version} + test org.apache.seatunnel datasource-jdbc-mysql - 1.0.0-SNAPSHOT - provided + ${project.version} + test org.apache.seatunnel datasource-mysql-cdc - 1.0.0-SNAPSHOT - provided - - - mysql - mysql-connector-java - ${mysql-connector.version} - provided + ${project.version} + test org.apache.seatunnel datasource-sqlserver-cdc - 1.0.0-SNAPSHOT - provided + ${project.version} + test org.apache.seatunnel datasource-jdbc-sqlserver - 1.0.0-SNAPSHOT - provided + ${project.version} + test @@ -773,56 +469,40 @@ - - org.awaitility - awaitility - compile - org.junit.jupiter junit-jupiter-api + test + + mysql + mysql-connector-java + ${mysql.version} + test + com.oracle.database.jdbc ojdbc8 ${oracle-jdbc.version} + test - - - com.clickhouse - clickhouse-jdbc - ${clickhouse.version} - - - org.postgresql postgresql ${postgresql.version} + test - - com.amazon.redshift - redshift-jdbc42 - ${redshift.version} - - - com.microsoft.sqlserver mssql-jdbc ${sqlserver.version} + test - - - mysql - mysql-connector-java - ${mysql-connector.version} - diff --git a/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh new file mode 100644 index 000000000..3ea36ab6d --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#This script is used to download the connector plug-ins required during the running process. +#All are downloaded by default. You can also choose what you need. +#You only need to configure the plug-in name in config/plugin_config. + +# get seatunnel web home +SEATUNNEL_WEB_HOME=$(cd $(dirname $0);cd ../;pwd) +DATASOURCE_DIR=${SEATUNNEL_WEB_HOME}/datasource + +# If you don’t want to download a certain data source, you can delete the element below +datasource_list=( + "datasource-plugins-api" + "datasource-elasticsearch" + "datasource-hive" + "datasource-jdbc-clickhouse" + "datasource-jdbc-hive" + "datasource-jdbc-mysql" + "datasource-jdbc-oracle" + "datasource-jdbc-postgresql" + "datasource-jdbc-redshift" + "datasource-jdbc-sqlserver" + "datasource-jdbc-starrocks" + "datasource-jdbc-tidb" + "datasource-kafka" + "datasource-mysql-cdc" + "datasource-s3" + "datasource-sqlserver-cdc" + "datasource-starrocks" + "datasource-mongodb" +) + +# the datasource default version is 1.0.0, you can also choose a custom version. eg: 1.1.2: sh install-datasource.sh 2.1.2 +version=1.0.0 + +if [ -n "$1" ]; then + version="$1" +fi + +echo "Downloading SeaTunnel Web Datasource lib, usage version is ${version}" + +# create the datasource directory +if [ ! -d "$DATASOURCE_DIR" ]; + then + mkdir -p "$DATASOURCE_DIR" + echo "Created datasource directory." +fi + +for i in "${datasource_list[@]}" +do + echo "$i" + echo "Downloading datasource: " "$i" + "$SEATUNNEL_WEB_HOME"/mvnw dependency:get -DgroupId=org.apache.seatunnel -DartifactId="$i" -Dversion="$version" -Ddest="$DATASOURCE_DIR" +done diff --git a/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh b/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh old mode 100644 new mode 100755 index eef62beb7..12f155b68 --- a/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh +++ b/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh @@ -22,10 +22,23 @@ set - WORKDIR=$(cd "$(dirname "$0")" || exit; pwd) +# check +check() { + # check whether the SEATUNNEL_HOME exists or not. + if [ "${SEATUNNEL_HOME}" ];then + echo "Load connectors from ${SEATUNNEL_HOME}" + else + echo "SEATUNNEL_HOME is not be set. Please check it." + exit 1 + fi +} + # start start() { echo "starting seatunnel..." + check + JAVA_OPTS="${JAVA_OPTS} -server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof" SPRING_OPTS="${SPRING_OPTS} -Dspring.config.name=application.yml -Dspring.config.location=classpath:application.yml" JAVA_OPTS="${JAVA_OPTS} -Dseatunnel-web.logs.path=${WORKDIR}/../logs" @@ -78,4 +91,4 @@ case "$1" in *) echo "Usage: seatunnel-daemon.sh {start|stop|status}" exit 1 -esac \ No newline at end of file +esac diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java index 791d12b41..1b6040bc2 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java @@ -25,8 +25,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.transaction.annotation.EnableTransactionManagement; -@SpringBootApplication( - scanBasePackages = {"org.apache.seatunnel.app", "org.apache.seatunnel.scheduler"}) +@SpringBootApplication(scanBasePackages = {"org.apache.seatunnel.app"}) @EnableTransactionManagement @EnableConfigurationProperties @EnableScheduling diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java index 214f81391..061453f32 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/bean/engine/EngineDataType.java @@ -16,6 +16,8 @@ */ package org.apache.seatunnel.app.bean.engine; +import org.apache.seatunnel.api.table.catalog.DataTypeConvertException; +import org.apache.seatunnel.api.table.catalog.DataTypeConvertor; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -23,48 +25,88 @@ import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import lombok.AllArgsConstructor; -import lombok.Data; +import lombok.Getter; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public class EngineDataType { - public static DataType T_STRING = new DataType("string", BasicType.STRING_TYPE); - public static DataType T_BOOLEAN = new DataType("boolean", BasicType.BOOLEAN_TYPE); - public static DataType T_BYTE = new DataType("tinyint", BasicType.BYTE_TYPE); - public static DataType T_SHORT = new DataType("smallint", BasicType.SHORT_TYPE); - public static DataType T_INT = new DataType("int", BasicType.INT_TYPE); - public static DataType T_LONG = new DataType("bigint", BasicType.LONG_TYPE); - public static DataType T_FLOAT = new DataType("float", BasicType.FLOAT_TYPE); - public static DataType T_DOUBLE = new DataType("double", BasicType.DOUBLE_TYPE); - public static DataType T_VOID = new DataType("null", BasicType.VOID_TYPE); - - public static DataType T_DECIMAL = new DataType("decimal(38, 18)", new DecimalType(38, 18)); - - public static DataType T_LOCAL_DATE = new DataType("date", LocalTimeType.LOCAL_DATE_TYPE); - public static DataType T_LOCAL_TIME = new DataType("time", LocalTimeType.LOCAL_TIME_TYPE); - public static DataType T_LOCAL_DATE_TIME = - new DataType("timestamp", LocalTimeType.LOCAL_DATE_TIME_TYPE); - - public static DataType T_PRIMITIVE_BYTE_ARRAY = - new DataType("bytes", PrimitiveByteArrayType.INSTANCE); - - public static DataType T_STRING_ARRAY = - new DataType("array", ArrayType.STRING_ARRAY_TYPE); - public static DataType T_BOOLEAN_ARRAY = - new DataType("array", ArrayType.BOOLEAN_ARRAY_TYPE); - public static DataType T_BYTE_ARRAY = new DataType("array", ArrayType.BYTE_ARRAY_TYPE); - public static DataType T_SHORT_ARRAY = - new DataType("array", ArrayType.SHORT_ARRAY_TYPE); - public static DataType T_INT_ARRAY = new DataType("array", ArrayType.INT_ARRAY_TYPE); - public static DataType T_LONG_ARRAY = new DataType("array", ArrayType.LONG_ARRAY_TYPE); - public static DataType T_FLOAT_ARRAY = new DataType("array", ArrayType.FLOAT_ARRAY_TYPE); - public static DataType T_DOUBLE_ARRAY = - new DataType("array", ArrayType.DOUBLE_ARRAY_TYPE); - - @Data - @AllArgsConstructor - public static class DataType { - String name; - SeaTunnelDataType RawType; + private static final Map DATA_TYPE_MAP = + Arrays.stream(DataType.values()) + .collect(Collectors.toMap(DataType::getName, Function.identity())); + + public static List getAllDataType() { + return Arrays.asList(DataType.values()); + } + + public enum DataType { + T_STRING("string", BasicType.STRING_TYPE), + T_BOOLEAN("boolean", BasicType.BOOLEAN_TYPE), + T_BYTE("tinyint", BasicType.BYTE_TYPE), + T_SHORT("smallint", BasicType.SHORT_TYPE), + T_INT("int", BasicType.INT_TYPE), + T_LONG("bigint", BasicType.LONG_TYPE), + T_FLOAT("float", BasicType.FLOAT_TYPE), + T_DOUBLE("double", BasicType.DOUBLE_TYPE), + T_VOID("null", BasicType.VOID_TYPE), + + T_DECIMAL("decimal(38, 18)", new DecimalType(38, 18)), + + T_LOCAL_DATE("date", LocalTimeType.LOCAL_DATE_TYPE), + T_LOCAL_TIME("time", LocalTimeType.LOCAL_TIME_TYPE), + T_LOCAL_DATE_TIME("timestamp", LocalTimeType.LOCAL_DATE_TIME_TYPE), + + T_PRIMITIVE_BYTE_ARRAY("bytes", PrimitiveByteArrayType.INSTANCE), + + T_STRING_ARRAY("array", ArrayType.STRING_ARRAY_TYPE), + T_BOOLEAN_ARRAY("array", ArrayType.BOOLEAN_ARRAY_TYPE), + T_BYTE_ARRAY("array", ArrayType.BYTE_ARRAY_TYPE), + T_SHORT_ARRAY("array", ArrayType.SHORT_ARRAY_TYPE), + T_INT_ARRAY("array", ArrayType.INT_ARRAY_TYPE), + T_LONG_ARRAY("array", ArrayType.LONG_ARRAY_TYPE), + T_FLOAT_ARRAY("array", ArrayType.FLOAT_ARRAY_TYPE), + T_DOUBLE_ARRAY("array", ArrayType.DOUBLE_ARRAY_TYPE); + + @Getter private final String name; + @Getter private final SeaTunnelDataType RawType; + + DataType(String name, SeaTunnelDataType rawType) { + this.name = name; + this.RawType = rawType; + } + } + + /** This convertor is used to transform the data type from engine to connector. */ + public static class SeaTunnelDataTypeConvertor + implements DataTypeConvertor> { + + @Override + public SeaTunnelDataType toSeaTunnelType(String engineDataType) { + return DATA_TYPE_MAP.get(engineDataType.toLowerCase(Locale.ROOT)).getRawType(); + } + + @Override + public SeaTunnelDataType toSeaTunnelType( + SeaTunnelDataType seaTunnelDataType, Map map) + throws DataTypeConvertException { + return seaTunnelDataType; + } + + @Override + public SeaTunnelDataType toConnectorType( + SeaTunnelDataType seaTunnelDataType, Map map) + throws DataTypeConvertException { + return seaTunnelDataType; + } + + @Override + public String getIdentity() { + return "EngineDataTypeConvertor"; + } } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java index 0f081f2dd..655507ea0 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/EngineController.java @@ -29,7 +29,6 @@ import javax.annotation.Resource; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -49,7 +48,7 @@ public Result> listSupportEngines() { @ApiOperation(value = "list all supported Data Type", httpMethod = "GET") public Result> listSupportDataTypes() { return Result.success( - Arrays.stream(engineService.listSupportDataTypes()) + engineService.listSupportDataTypes().stream() .map(EngineDataType.DataType::getName) .collect(Collectors.toList())); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java index e95baa8e2..00f0120ae 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.app.domain.response.PageInfo; import org.apache.seatunnel.app.domain.response.job.JobDefinitionRes; import org.apache.seatunnel.app.service.IJobDefinitionService; +import org.apache.seatunnel.app.service.IJobTaskService; import org.apache.seatunnel.server.common.CodeGenerateUtils; import org.springframework.web.bind.annotation.DeleteMapping; @@ -46,6 +47,8 @@ public class JobDefinitionController { @Resource private IJobDefinitionService jobService; + @Resource private IJobTaskService jobTask; + /** * create job definition * @@ -81,6 +84,7 @@ Result getJobDefinition(@PathVariable long jobId) { Result deleteJobDefinition( @ApiParam(value = "id", required = true) @RequestParam long id) { jobService.deleteJob(id); + jobTask.deleteTaskByVersionId(id); return Result.success(); } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobExecutorController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobExecutorController.java index 23230201c..1a41e15b6 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobExecutorController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobExecutorController.java @@ -24,7 +24,6 @@ import org.apache.seatunnel.server.common.SeatunnelErrorEnum; import org.apache.seatunnel.server.common.SeatunnelException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestMapping; @@ -44,12 +43,12 @@ @RestController public class JobExecutorController { - @Autowired IJobExecutorService jobExecutorService; + @Resource IJobExecutorService jobExecutorService; @Resource private IJobInstanceService jobInstanceService; @GetMapping("/execute") @ApiOperation(value = "Execute synchronization tasks", httpMethod = "GET") - public Result jobExecutor( + public Result jobExecutor( @ApiParam(value = "userId", required = true) @RequestAttribute("userId") Integer userId, @ApiParam(value = "jobDefineId", required = true) @RequestParam("jobDefineId") Long jobDefineId) { @@ -73,14 +72,14 @@ public Result resource( } @GetMapping("/pause") - public Result jobPause( + public Result jobPause( @ApiParam(value = "userId", required = true) @RequestAttribute("userId") Integer userId, @ApiParam(value = "jobInstanceId", required = true) @RequestParam Long jobInstanceId) { return jobExecutorService.jobPause(userId, jobInstanceId); } @GetMapping("/restore") - public Result jobRestore( + public Result jobRestore( @ApiParam(value = "userId", required = true) @RequestAttribute("userId") Integer userId, @ApiParam(value = "jobInstanceId", required = true) @RequestParam Long jobInstanceId) { return jobExecutorService.jobStore(userId, jobInstanceId); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SchemaDerivationController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SchemaDerivationController.java new file mode 100644 index 000000000..03d2afb07 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SchemaDerivationController.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.controller; + +import org.apache.seatunnel.app.common.Result; +import org.apache.seatunnel.app.domain.request.job.TableSchemaReq; +import org.apache.seatunnel.app.domain.request.job.transform.SQL; +import org.apache.seatunnel.app.service.ISchemaDerivationService; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.ApiParam; + +import javax.annotation.Resource; + +@RestController +@RequestMapping("/seatunnel/api/v1/schema/derivation") +public class SchemaDerivationController { + + @Resource private ISchemaDerivationService schemaDerivationService; + + @PostMapping("/sql") + Result SQLSchemaDerivation( + @ApiParam(value = "job version id", required = true) @RequestParam long jobVersionId, + @ApiParam(value = "inputPluginId", required = true) @RequestParam String inputPluginId, + @RequestBody SQL sql) { + return Result.success( + schemaDerivationService.derivationSQL(jobVersionId, inputPluginId, sql)); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java index ac8929964..bb6823125 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java @@ -20,8 +20,6 @@ import org.apache.seatunnel.app.common.Constants; import org.apache.seatunnel.app.common.Result; import org.apache.seatunnel.app.dal.dao.IUserDao; -import org.apache.seatunnel.app.dal.dao.TaskDefinitionDao; -import org.apache.seatunnel.app.dal.entity.TaskMainInfo; import org.apache.seatunnel.app.dal.entity.User; import org.apache.seatunnel.app.domain.dto.datasource.DatabaseTableFields; import org.apache.seatunnel.app.domain.dto.datasource.DatabaseTables; @@ -77,8 +75,6 @@ public class SeatunnelDatasourceController extends BaseController { @Autowired private IDatasourceService datasourceService; - @Autowired private TaskDefinitionDao taskDefinitionDao; - @Resource(name = "userDaoImpl") private IUserDao userMapper; @@ -203,19 +199,6 @@ Result deleteDatasource( @ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser, @PathVariable("id") String id) { Long datasourceId = Long.parseLong(id); - List taskMainInfos = taskDefinitionDao.queryByDataSourceId(datasourceId); - if (taskMainInfos.size() > 0) { - throw new SeatunnelException( - SeatunnelErrorEnum.DATA_SOURCE_HAD_USED, - taskMainInfos.stream() - .map( - info -> - String.format( - "%s - %s", - info.getProcessDefinitionName(), - info.getTaskName())) - .collect(Collectors.toList())); - } return Result.success(datasourceService.deleteDatasource(loginUser.getId(), datasourceId)); } @@ -354,8 +337,11 @@ Result> getDatabases( Result> getTableNames( @ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser, @RequestParam("datasourceName") String datasourceName, - @RequestParam("databaseName") String databaseName) { - return Result.success(datasourceService.queryTableNames(datasourceName, databaseName)); + @RequestParam("databaseName") String databaseName, + @RequestParam("filterName") String filterName, + @RequestParam("size") Integer size) { + return Result.success( + datasourceService.queryTableNames(datasourceName, databaseName, filterName, size)); } @GetMapping("/schema") diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskInstanceController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskInstanceController.java index a56e56f1a..cdcd35a2f 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskInstanceController.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskInstanceController.java @@ -19,8 +19,8 @@ import org.apache.seatunnel.app.common.Result; import org.apache.seatunnel.app.domain.dto.job.SeaTunnelJobInstanceDto; -import org.apache.seatunnel.app.domain.response.PageInfo; import org.apache.seatunnel.app.service.ITaskInstanceService; +import org.apache.seatunnel.app.utils.PageInfo; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; @@ -35,7 +35,7 @@ @RestController public class TaskInstanceController { - @Autowired ITaskInstanceService taskInstanceService; + @Autowired ITaskInstanceService taskInstanceService; @GetMapping("/jobMetrics") @ApiOperation(value = "get the jobMetrics list ", httpMethod = "GET") @@ -49,18 +49,15 @@ public Result> getTaskInstanceList( @RequestParam("syncTaskType") String syncTaskType, @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize) { - Result> result = - taskInstanceService.getSyncTaskInstancePaging( - userId, - jobDefineName, - executorName, - stateType, - startTime, - endTime, - syncTaskType, - pageNo, - pageSize); - - return result; + return taskInstanceService.getSyncTaskInstancePaging( + userId, + jobDefineName, + executorName, + stateType, + startTime, + endTime, + syncTaskType, + pageNo, + pageSize); } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java index 9136c5926..27224c966 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobInstanceHistoryDao.java @@ -23,4 +23,6 @@ public interface IJobInstanceHistoryDao { JobInstanceHistory getByInstanceId(Long jobInstanceId); void insert(JobInstanceHistory jobInstanceHistory); + + void updateJobInstanceHistory(JobInstanceHistory jobInstanceHistory); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobTaskDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobTaskDao.java index f9ae0a14d..ffc9c6c09 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobTaskDao.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IJobTaskDao.java @@ -38,4 +38,6 @@ public interface IJobTaskDao { void deleteTasks(List jobTaskIds); void deleteTask(long jobVersionId, String pluginId); + + void deleteTaskByVersionId(long id); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java index 3655b0219..1b440e2f1 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobInstanceHistoryDaoImpl.java @@ -43,4 +43,9 @@ public JobInstanceHistory getByInstanceId(Long jobInstanceId) { public void insert(JobInstanceHistory jobInstanceHistory) { jobInstanceHistoryMapper.insert(jobInstanceHistory); } + + @Override + public void updateJobInstanceHistory(JobInstanceHistory jobInstanceHistory) { + jobInstanceHistoryMapper.updateById(jobInstanceHistory); + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobTaskDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobTaskDaoImpl.java index 540e4c72b..5ab86f591 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobTaskDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobTaskDaoImpl.java @@ -87,4 +87,9 @@ public void deleteTask(long jobVersionId, String pluginId) { .eq(JobTask::getVersionId, jobVersionId) .and(i -> i.eq(JobTask::getPluginId, pluginId))); } + + @Override + public void deleteTaskByVersionId(long id) { + jobTaskMapper.delete(Wrappers.lambdaQuery(new JobTask()).eq(JobTask::getVersionId, id)); + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobVersionDaoImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobVersionDaoImpl.java index 4f6c52d96..4c1dc1547 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobVersionDaoImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/JobVersionDaoImpl.java @@ -55,7 +55,7 @@ public JobVersion getLatestVersion(long jobId) { @Override public List getLatestVersionByJobIds(List jobIds) { - QueryWrapper wrapper = new QueryWrapper(); + QueryWrapper wrapper = new QueryWrapper<>(); wrapper.in("job_id", jobIds); return jobVersionMapper.selectList(wrapper); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java index ed758cd13..b4e3b3848 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IEngineService.java @@ -25,5 +25,5 @@ public interface IEngineService { List listSupportEngines(); - EngineDataType.DataType[] listSupportDataTypes(); + List listSupportDataTypes(); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobExecutorService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobExecutorService.java index 1d2b1c579..01ce17e59 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobExecutorService.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobExecutorService.java @@ -21,9 +21,9 @@ public interface IJobExecutorService { - public Result jobExecute(Integer userId, Long jobDefineId); + Result jobExecute(Integer userId, Long jobDefineId); - public Result jobPause(Integer userId, Long jobInstanceId); + Result jobPause(Integer userId, Long jobInstanceId); - public Result jobStore(Integer userId, Long jobInstanceId); + Result jobStore(Integer userId, Long jobInstanceId); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java index 71c7d30f4..7e213e3b0 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IJobTaskService.java @@ -36,4 +36,6 @@ public interface IJobTaskService { T getTransformOptions(long jobVersionId, String pluginId); void deleteSingleTask(long jobVersionId, String pluginId); + + void deleteTaskByVersionId(long id); } diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ComplementDataDto.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ISchemaDerivationService.java similarity index 72% rename from seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ComplementDataDto.java rename to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ISchemaDerivationService.java index f1f10dcc1..922f24d8f 100644 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/ComplementDataDto.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ISchemaDerivationService.java @@ -15,17 +15,12 @@ * limitations under the License. */ -package org.apache.seatunnel.spi.scheduler.dto; +package org.apache.seatunnel.app.service; -import lombok.Builder; -import lombok.Data; +import org.apache.seatunnel.app.domain.request.job.TableSchemaReq; +import org.apache.seatunnel.app.domain.request.job.transform.SQL; -import java.util.Date; +public interface ISchemaDerivationService { -@Data -@Builder -public class ComplementDataDto { - private Date startTime; - private Date endTime; - private Integer parallelismNum; + TableSchemaReq derivationSQL(long jobVersionId, String inputPluginId, SQL sql); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ITaskInstanceService.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ITaskInstanceService.java index 644a75b3e..77f2d1150 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ITaskInstanceService.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/ITaskInstanceService.java @@ -18,10 +18,11 @@ package org.apache.seatunnel.app.service; import org.apache.seatunnel.app.common.Result; +import org.apache.seatunnel.app.utils.PageInfo; -public interface ITaskInstanceService { +public interface ITaskInstanceService { - Result getSyncTaskInstancePaging( + Result> getSyncTaskInstancePaging( Integer userId, String jobDefineName, String executorName, diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java index 6f4cfd60b..4e290fc0a 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/DatasourceServiceImpl.java @@ -20,8 +20,10 @@ import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.app.config.ConnectorDataSourceMapperConfig; import org.apache.seatunnel.app.dal.dao.IDatasourceDao; +import org.apache.seatunnel.app.dal.dao.IJobTaskDao; import org.apache.seatunnel.app.dal.dao.IVirtualTableDao; import org.apache.seatunnel.app.dal.entity.Datasource; +import org.apache.seatunnel.app.dal.entity.JobTask; import org.apache.seatunnel.app.dal.entity.VirtualTable; import org.apache.seatunnel.app.domain.response.PageInfo; import org.apache.seatunnel.app.domain.response.datasource.DatasourceDetailRes; @@ -81,6 +83,9 @@ public class DatasourceServiceImpl extends SeatunnelBaseServiceImpl @Resource private IJobDefinitionService jobDefinitionService; + @Resource(name = "jobTaskDaoImpl") + private IJobTaskDao jobTaskDao; + @Autowired @Qualifier("virtualTableDaoImpl") private IVirtualTableDao virtualTableDao; @@ -180,6 +185,11 @@ public boolean deleteDatasource(Integer userId, Long datasourceId) { SeatunnelResourcePermissionModuleEnum.DATASOURCE.name(), Collections.singletonList(datasourceId), userId); + // check has job task has used this datasource + List jobTaskList = jobTaskDao.getJobTaskByDataSourceId(datasourceId); + if (!CollectionUtils.isEmpty(jobTaskList)) { + throw new SeatunnelException(SeatunnelErrorEnum.DATA_SOURCE_HAD_USED); + } // check has virtual table has used this datasource List virtualDatabaseNames = virtualTableDao.getVirtualDatabaseNames(datasourceId); if (!CollectionUtils.isEmpty(virtualDatabaseNames)) { diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java index b4a6aa970..48da062f1 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/EngineServiceImpl.java @@ -45,31 +45,8 @@ public List listSupportEngines() { } @Override - public EngineDataType.DataType[] listSupportDataTypes() { + public List listSupportDataTypes() { funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.ENGIN_TYPE, 0); - return new EngineDataType.DataType[] { - EngineDataType.T_STRING, - EngineDataType.T_BOOLEAN, - EngineDataType.T_BYTE, - EngineDataType.T_SHORT, - EngineDataType.T_INT, - EngineDataType.T_LONG, - EngineDataType.T_FLOAT, - EngineDataType.T_DOUBLE, - EngineDataType.T_VOID, - EngineDataType.T_DECIMAL, - EngineDataType.T_LOCAL_DATE, - EngineDataType.T_LOCAL_TIME, - EngineDataType.T_LOCAL_DATE_TIME, - EngineDataType.T_PRIMITIVE_BYTE_ARRAY, - EngineDataType.T_STRING_ARRAY, - EngineDataType.T_BOOLEAN_ARRAY, - EngineDataType.T_BYTE_ARRAY, - EngineDataType.T_SHORT_ARRAY, - EngineDataType.T_INT_ARRAY, - EngineDataType.T_LONG_ARRAY, - EngineDataType.T_FLOAT_ARRAY, - EngineDataType.T_DOUBLE_ARRAY - }; + return EngineDataType.getAllDataType(); } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobExecutorServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobExecutorServiceImpl.java index 797d5d38a..947519f1f 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobExecutorServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobExecutorServiceImpl.java @@ -29,6 +29,7 @@ import org.apache.seatunnel.app.thirdparty.metrics.IEngineMetricsExtractor; import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.common.config.DeployMode; +import org.apache.seatunnel.common.utils.ExceptionUtils; import org.apache.seatunnel.engine.client.SeaTunnelClient; import org.apache.seatunnel.engine.client.job.ClientJobProxy; import org.apache.seatunnel.engine.client.job.JobExecutionEnvironment; @@ -48,6 +49,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -60,7 +62,7 @@ public class JobExecutorServiceImpl implements IJobExecutorService { @Resource private IJobInstanceDao jobInstanceDao; @Override - public Result jobExecute(Integer userId, Long jobDefineId) { + public Result jobExecute(Integer userId, Long jobDefineId) { JobExecutorRes executeResource = jobInstanceService.createExecuteResource(userId, jobDefineId); @@ -75,7 +77,8 @@ public Result jobExecute(Integer userId, Long jobDefineId) { public String writeJobConfigIntoConfFile(String jobConfig, Long jobDefineId) { String projectRoot = System.getProperty("user.dir"); - String filePath = projectRoot + "\\profile\\" + Long.toString(jobDefineId) + ".conf"; + String filePath = + projectRoot + File.separator + "profile" + File.separator + jobDefineId + ".conf"; try { File file = new File(filePath); if (!file.exists()) { @@ -90,7 +93,7 @@ public String writeJobConfigIntoConfFile(String jobConfig, Long jobDefineId) { log.info("File created and content written successfully."); } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException(e); } return filePath; } @@ -119,6 +122,7 @@ public Long executeJobBySeaTunnel(Integer userId, String filePath, Long jobInsta }); } catch (ExecutionException | InterruptedException e) { + ExceptionUtils.getMessage(e); throw new RuntimeException(e); } return jobInstanceId; @@ -132,23 +136,20 @@ public void waitJobFinish( SeaTunnelClient seaTunnelClient) { ExecutorService executor = Executors.newFixedThreadPool(1); CompletableFuture future = - CompletableFuture.supplyAsync( - () -> { - return clientJobProxy.waitForJobComplete(); - }, - executor); + CompletableFuture.supplyAsync(clientJobProxy::waitForJobComplete, executor); try { + log.info("future.get before"); JobStatus jobStatus = future.get(); - if (JobStatus.FINISHED.equals(jobStatus)) { - jobInstanceService.complete(userId, jobInstanceId, jobEngineId); - executor.shutdown(); - } + + executor.shutdown(); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ExecutionException e) { throw new RuntimeException(e); } finally { seaTunnelClient.close(); + log.info("and jobInstanceService.complete begin"); + jobInstanceService.complete(userId, jobInstanceId, jobEngineId); } } @@ -159,14 +160,14 @@ private SeaTunnelClient createSeaTunnelClient() { } public static String getClusterName(String testClassName) { - // return System.getProperty("user.name") + "_" + testClassName; return testClassName; } @Override - public Result jobPause(Integer userId, Long jobInstanceId) { + public Result jobPause(Integer userId, Long jobInstanceId) { JobInstance jobInstance = jobInstanceDao.getJobInstance(jobInstanceId); - if (getJobStatusFromEngine(jobInstance, jobInstance.getJobEngineId()) == "RUNNING") { + if (Objects.equals( + getJobStatusFromEngine(jobInstance, jobInstance.getJobEngineId()), "RUNNING")) { pauseJobInEngine(jobInstance.getJobEngineId()); } return Result.success(); @@ -187,13 +188,18 @@ private void pauseJobInEngine(@NonNull String jobEngineId) { } @Override - public Result jobStore(Integer userId, Long jobInstanceId) { + public Result jobStore(Integer userId, Long jobInstanceId) { JobInstance jobInstance = jobInstanceDao.getJobInstance(jobInstanceId); String projectRoot = System.getProperty("user.dir"); String filePath = - projectRoot + "\\profile\\" + Long.toString(jobInstance.getJobDefineId()) + ".conf"; - + projectRoot + + File.separator + + "profile" + + File.separator + + jobInstance.getJobDefineId() + + ".conf"; + log.info("jobStore filePath:{}", filePath); SeaTunnelEngineProxy.getInstance() .restoreJob(filePath, jobInstanceId, Long.valueOf(jobInstance.getJobEngineId())); return Result.success(); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java index c2d1a786e..92726a660 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java @@ -421,8 +421,6 @@ private List findInputSchemas( } catch (JsonProcessingException e) { throw new SeatunnelException(SeatunnelErrorEnum.ILLEGAL_STATE, e.getMessage()); - } catch (IOException e) { - throw new RuntimeException(e); } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java index 47fa7a63e..837de0061 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobMetricsServiceImpl.java @@ -53,10 +53,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -101,10 +99,8 @@ public JobSummaryMetricsRes getJobSummaryMetrics( long writeCount = jobPipelineDetailMetrics.stream().mapToLong(JobMetrics::getWriteRowCount).sum(); - JobSummaryMetricsRes jobSummaryMetricsRes = - new JobSummaryMetricsRes( - jobInstanceId, Long.valueOf(jobEngineId), readCount, writeCount, jobStatus); - return jobSummaryMetricsRes; + return new JobSummaryMetricsRes( + jobInstanceId, Long.parseLong(jobEngineId), readCount, writeCount, jobStatus); } @Override @@ -117,6 +113,12 @@ public Map getALLJobSummaryMetrics( funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_METRICS_SUMMARY, userId); List allJobInstance = jobInstanceDao.getAllJobInstance(jobInstanceIdList); + if (allJobInstance.isEmpty()) { + log.warn( + "getALLJobSummaryMetrics : allJobInstance is empty, task id list is {}", + jobInstanceIdList); + return new HashMap<>(); + } Map result = null; Map> allRunningJobMetricsFromEngine = getAllRunningJobMetricsFromEngine( @@ -140,7 +142,7 @@ public Map getALLJobSummaryMetrics( jobInstanceIdAndJobEngineIdMap); } - log.info("result is {}", result.toString()); + log.info("result is {}", result == null ? "null" : result.toString()); return result; } @@ -181,32 +183,35 @@ private Map getMatricsListIfTaskTypeIsBatch( } else { log.info( "The job does not exist on the engine, it is directly returned from the database"); - JobSummaryMetricsRes jobMetriceFromDb = + JobSummaryMetricsRes jobMetricsFromDb = getJobSummaryMetricsResByDb( jobInstance, userId, Long.toString( jobInstanceIdAndJobEngineIdMap.get( jobInstance.getId()))); - if (jobMetriceFromDb != null) { - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetriceFromDb); + if (jobMetricsFromDb != null) { + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromDb); + } + if ("RUNNING".equals(jobInstance.getJobStatus())) { + // Set the job status of jobInstance and jobMetrics in the database to + // finished + jobInstance.setJobStatus("FINISHED"); + jobInstanceDao.getJobInstanceMapper().updateById(jobInstance); } - // 将数据库中的jobInstance和jobMetrics的作业状态改为finished - jobInstance.setJobStatus("FINISHED"); - jobInstanceDao.getJobInstanceMapper().updateById(jobInstance); } } else if (jobInstance.getJobStatus().equals("FINISHED") || jobInstance.getJobStatus().equals("CANCELED")) { // If the status of the job is finished or cancelled, the monitoring information is // directly obtained from MySQL - JobSummaryMetricsRes jobMetriceFromDb = + JobSummaryMetricsRes jobMetricsFromDb = getJobSummaryMetricsResByDb( jobInstance, userId, Long.toString( jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))); - log.info("jobStatus=finish oe canceled,JobSummaryMetricsRes={}", jobMetriceFromDb); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetriceFromDb); + log.info("jobStatus=finish oe canceled,JobSummaryMetricsRes={}", jobMetricsFromDb); + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromDb); } } @@ -230,21 +235,19 @@ private void modifyAndUpdateJobInstanceAndJobMetrics( syncMetricsToDbRunning(jobInstance, userId, jobMetricsFromEngine); jobInstanceDao.update(jobInstance); } else { - jobMetricsFromDb.stream() - .forEach( - jobMetrics -> - jobMetrics.setReadRowCount( - jobMetricsFromEngine - .get(jobMetrics.getPipelineId()) - .getReadRowCount())); - jobMetricsFromDb.stream() - .forEach( - jobMetrics -> - jobMetrics.setWriteRowCount( - jobMetricsFromEngine - .get(jobMetrics.getPipelineId()) - .getWriteRowCount())); - jobMetricsFromDb.stream().forEach(jobMetrics -> jobMetrics.setStatus("RUNNING")); + jobMetricsFromDb.forEach( + jobMetrics -> + jobMetrics.setReadRowCount( + jobMetricsFromEngine + .get(jobMetrics.getPipelineId()) + .getReadRowCount())); + jobMetricsFromDb.forEach( + jobMetrics -> + jobMetrics.setWriteRowCount( + jobMetricsFromEngine + .get(jobMetrics.getPipelineId()) + .getWriteRowCount())); + jobMetricsFromDb.forEach(jobMetrics -> jobMetrics.setStatus("RUNNING")); updateJobInstanceAndMetrics(jobInstance, jobMetricsFromDb); } @@ -261,105 +264,121 @@ private Map getMatricsListIfTaskTypeIsStreaming( // Traverse all jobInstances in allJobInstance for (JobInstance jobInstance : allJobInstance) { - if (jobInstance.getJobStatus() != null - && jobInstance.getJobStatus().equals("CANCELED")) { - // If the status of the job is finished or cancelled - // the monitoring information is directly obtained from MySQL - JobSummaryMetricsRes jobMetriceFromDb = - getJobSummaryMetricsResByDb( - jobInstance, - userId, - Long.toString( - jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetriceFromDb); - - } else if (jobInstance.getJobStatus() != null - && (jobInstance.getJobStatus().equals("FINISHED") - || jobInstance.getJobStatus().equals("FAILED"))) { - // Obtain monitoring information from the collection of running jobs returned from - // the engine - if (!allRunningJobMetricsFromEngine.isEmpty() - && allRunningJobMetricsFromEngine.containsKey( - jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))) { - // If it can be found, update the information in MySQL and return it to the - // front-end data - modifyAndUpdateJobInstanceAndJobMetrics( - jobInstance, - allRunningJobMetricsFromEngine, - jobInstanceIdAndJobEngineIdMap, - userId); - - /** Return data from the front-end */ - JobSummaryMetricsRes jobMetricsFromEngineRes = - getRunningJobMetricsFromEngine( - allRunningJobMetricsFromEngine, - jobInstanceIdAndJobEngineIdMap, - jobInstance); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromEngineRes); - } else { - // If not found, obtain information from MySQL - JobSummaryMetricsRes jobMetriceFromDb = + try { + if (jobInstance.getJobStatus() != null + && jobInstance.getJobStatus().equals("CANCELED")) { + // If the status of the job is finished or cancelled + // the monitoring information is directly obtained from MySQL + JobSummaryMetricsRes jobMetricsFromDb = getJobSummaryMetricsResByDb( jobInstance, userId, Long.toString( jobInstanceIdAndJobEngineIdMap.get( jobInstance.getId()))); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetriceFromDb); - } - } else { - // Obtain monitoring information from the collection of running jobs returned from - // the engine - if (!allRunningJobMetricsFromEngine.isEmpty() - && allRunningJobMetricsFromEngine.containsKey( - jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))) { - modifyAndUpdateJobInstanceAndJobMetrics( - jobInstance, - allRunningJobMetricsFromEngine, - jobInstanceIdAndJobEngineIdMap, - userId); - /** Return data from the front-end */ - JobSummaryMetricsRes jobMetricsFromEngineRes = - getRunningJobMetricsFromEngine( - allRunningJobMetricsFromEngine, - jobInstanceIdAndJobEngineIdMap, - jobInstance); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromEngineRes); - } else { - String jobStatusByJobEngineId = - getJobStatusByJobEngineId( - String.valueOf( - jobInstanceIdAndJobEngineIdMap.get( - jobInstance.getId()))); - if (jobStatusByJobEngineId != null) { - jobInstance.setJobStatus(jobStatusByJobEngineId); - jobInstanceDao.update(jobInstance); - JobSummaryMetricsRes jobSummaryMetricsResByDb = + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromDb); + + } else if (jobInstance.getJobStatus() != null + && (jobInstance.getJobStatus().equals("FINISHED") + || jobInstance.getJobStatus().equals("FAILED"))) { + // Obtain monitoring information from the collection of running jobs returned + // from + // the engine + if (!allRunningJobMetricsFromEngine.isEmpty() + && allRunningJobMetricsFromEngine.containsKey( + jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))) { + // If it can be found, update the information in MySQL and return it to the + // front-end data + modifyAndUpdateJobInstanceAndJobMetrics( + jobInstance, + allRunningJobMetricsFromEngine, + jobInstanceIdAndJobEngineIdMap, + userId); + + // Return data from the front-end + JobSummaryMetricsRes jobMetricsFromEngineRes = + getRunningJobMetricsFromEngine( + allRunningJobMetricsFromEngine, + jobInstanceIdAndJobEngineIdMap, + jobInstance); + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromEngineRes); + } else { + // If not found, obtain information from MySQL + JobSummaryMetricsRes jobMetricsFromDb = getJobSummaryMetricsResByDb( jobInstance, userId, - String.valueOf( - jobInstanceIdAndJobEngineIdMap.get( - jobInstance.getId()))); - jobSummaryMetricsResMap.put(jobInstance.getId(), jobSummaryMetricsResByDb); - List jobMetricsFromDb = - getJobMetricsFromDb( - jobInstance, - userId, - String.valueOf( + Long.toString( jobInstanceIdAndJobEngineIdMap.get( jobInstance.getId()))); - if (!jobMetricsFromDb.isEmpty()) { - jobMetricsFromDb.stream() - .forEach( - jobMetrics -> - jobMetrics.setStatus(jobStatusByJobEngineId)); - for (JobMetrics jobMetrics : jobMetricsFromDb) { - jobMetricsDao.getJobMetricsMapper().updateById(jobMetrics); + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromDb); + } + } else { + // Obtain monitoring information from the collection of running jobs returned + // from + // the engine + if (!allRunningJobMetricsFromEngine.isEmpty() + && allRunningJobMetricsFromEngine.containsKey( + jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId()))) { + modifyAndUpdateJobInstanceAndJobMetrics( + jobInstance, + allRunningJobMetricsFromEngine, + jobInstanceIdAndJobEngineIdMap, + userId); + // Return data from the front-end + JobSummaryMetricsRes jobMetricsFromEngineRes = + getRunningJobMetricsFromEngine( + allRunningJobMetricsFromEngine, + jobInstanceIdAndJobEngineIdMap, + jobInstance); + jobSummaryMetricsResMap.put(jobInstance.getId(), jobMetricsFromEngineRes); + } else { + String jobStatusByJobEngineId = null; + try { + jobStatusByJobEngineId = + getJobStatusByJobEngineId( + String.valueOf( + jobInstanceIdAndJobEngineIdMap.get( + jobInstance.getId()))); + } catch (Exception e) { + log.warn( + "getMetricsListIfTaskTypeIsStreaming getJobStatusByJobEngineId is exception jobInstanceId is : {}", + jobInstance.getId()); + } + + if (jobStatusByJobEngineId != null) { + jobInstance.setJobStatus(jobStatusByJobEngineId); + jobInstanceDao.update(jobInstance); + JobSummaryMetricsRes jobSummaryMetricsResByDb = + getJobSummaryMetricsResByDb( + jobInstance, + userId, + String.valueOf( + jobInstanceIdAndJobEngineIdMap.get( + jobInstance.getId()))); + jobSummaryMetricsResMap.put( + jobInstance.getId(), jobSummaryMetricsResByDb); + List jobMetricsFromDb = + getJobMetricsFromDb( + jobInstance, + userId, + String.valueOf( + jobInstanceIdAndJobEngineIdMap.get( + jobInstance.getId()))); + if (!jobMetricsFromDb.isEmpty()) { + String finalJobStatusByJobEngineId = jobStatusByJobEngineId; + jobMetricsFromDb.forEach( + jobMetrics -> + jobMetrics.setStatus(finalJobStatusByJobEngineId)); + for (JobMetrics jobMetrics : jobMetricsFromDb) { + jobMetricsDao.getJobMetricsMapper().updateById(jobMetrics); + } } } } } + } catch (Exception e) { + throw new RuntimeException(e); } } return jobSummaryMetricsResMap; @@ -375,27 +394,16 @@ private JobSummaryMetricsRes getRunningJobMetricsFromEngine( allRunningJobMetricsFromEngine.get( jobInstanceIdAndJobEngineIdMap.get(jobInstance.getId())); log.info("0706jobMetricsFromEngine={}", jobMetricsFromEngine); - long readCount = 0l; - long writeCount = 0l; - - Collection values = jobMetricsFromEngine.values(); - Iterator iterator = values.iterator(); - while (iterator.hasNext()) { - JobMetrics next = iterator.next(); - readCount += next.getReadRowCount(); - } - - writeCount = + long readCount = + jobMetricsFromEngine.values().stream().mapToLong(JobMetrics::getReadRowCount).sum(); + long writeCount = jobMetricsFromEngine.values().stream() .mapToLong(JobMetrics::getWriteRowCount) .sum(); - log.info("jobInstance={}", jobInstance.toString()); - - JobSummaryMetricsRes jobSummaryMetricsRes = - new JobSummaryMetricsRes(jobInstance.getId(), 1l, readCount, writeCount, "RUNNING"); + log.info("jobInstance={}", jobInstance); - return jobSummaryMetricsRes; + return new JobSummaryMetricsRes(jobInstance.getId(), 1L, readCount, writeCount, "RUNNING"); } private JobSummaryMetricsRes getJobSummaryMetricsResByDb( @@ -405,14 +413,12 @@ private JobSummaryMetricsRes getJobSummaryMetricsResByDb( long readCount = jobMetricsFromDb.stream().mapToLong(JobMetrics::getReadRowCount).sum(); long writeCount = jobMetricsFromDb.stream().mapToLong(JobMetrics::getWriteRowCount).sum(); - JobSummaryMetricsRes jobSummaryMetricsRes = - new JobSummaryMetricsRes( - jobInstance.getId(), - Long.valueOf(jobInstance.getJobEngineId()), - readCount, - writeCount, - jobInstance.getJobStatus()); - return jobSummaryMetricsRes; + return new JobSummaryMetricsRes( + jobInstance.getId(), + Long.parseLong(jobInstance.getJobEngineId()), + readCount, + writeCount, + jobInstance.getJobStatus()); } return null; } @@ -457,12 +463,14 @@ private List getJobPipelineDetailMetrics( @NonNull JobInstance jobInstance, @NonNull Integer userId, @NonNull String jobEngineId, - @NonNull String jobStatus, + String jobStatus, @NonNull IEngineMetricsExtractor engineMetricsExtractor) { // If job is not end state, get metrics from engine. - List jobMetrics = new ArrayList<>(); - if (engineMetricsExtractor.isJobEndStatus(jobStatus)) { + List jobMetrics; + if (jobStatus == null) { + jobMetrics = getJobMetricsFromDb(jobInstance, userId, jobEngineId); + } else if (engineMetricsExtractor.isJobEndStatus(jobStatus)) { jobMetrics = getJobMetricsFromDb(jobInstance, userId, jobEngineId); if (CollectionUtils.isEmpty(jobMetrics)) { syncMetricsToDb(jobInstance, userId, jobEngineId); @@ -496,10 +504,8 @@ private List getJobPipelineMetrics( IEngineMetricsExtractor engineMetricsExtractor = (new EngineMetricsExtractorFactory(engine)).getEngineMetricsExtractor(); String jobStatus = engineMetricsExtractor.getJobStatus(jobEngineId); - List jobPipelineDetailMetrics = - getJobPipelineDetailMetrics( - jobInstance, userId, jobEngineId, jobStatus, engineMetricsExtractor); - return jobPipelineDetailMetrics; + return getJobPipelineDetailMetrics( + jobInstance, userId, jobEngineId, jobStatus, engineMetricsExtractor); } @Override @@ -616,7 +622,13 @@ private void syncHistoryJobInfoToDb( jobHistoryFromEngine.setId(jobInstance.getId()); - jobInstanceHistoryDao.insert(jobHistoryFromEngine); + JobInstanceHistory byInstanceId = + jobInstanceHistoryDao.getByInstanceId(jobInstance.getId()); + if (byInstanceId == null) { + jobInstanceHistoryDao.insert(jobHistoryFromEngine); + } else { + jobInstanceHistoryDao.updateJobInstanceHistory(jobHistoryFromEngine); + } } private void syncCompleteJobInfoToDb(@NonNull JobInstance jobInstance) { diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java index 7e38a6bcf..b810ff923 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobTaskServiceImpl.java @@ -508,7 +508,7 @@ private void transformOptionCheck(String connectorType, String transformOptionsS SQLTransformOptions sqlTransformOptions = getTransformOption(transform, transformOptionsStr); if (sqlTransformOptions != null) { - // TODO 调用接口返回目标字段 + // TODO The call interface returns the target field List fields = new ArrayList<>(); checkTransformTargetFieldRepeat(fields); } @@ -554,6 +554,12 @@ public void deleteSingleTask(long jobVersionId, String pluginId) { jobTaskDao.deleteTask(jobVersionId, pluginId); } + @Override + public void deleteTaskByVersionId(long id) { + funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.SINGLE_TASK_DELETE, 0); + jobTaskDao.deleteTaskByVersionId(id); + } + private String getConnectorTypeFromDataSource(long datasourceId) { String pluginName = datasourceService diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/SchemaDerivationServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/SchemaDerivationServiceImpl.java new file mode 100644 index 000000000..03cacb2b3 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/SchemaDerivationServiceImpl.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.service.impl; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; +import org.apache.seatunnel.api.table.catalog.PhysicalColumn; +import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.api.table.connector.TableTransform; +import org.apache.seatunnel.api.table.factory.FactoryUtil; +import org.apache.seatunnel.api.table.factory.TableFactoryContext; +import org.apache.seatunnel.api.table.factory.TableTransformFactory; +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.app.domain.request.job.DatabaseTableSchemaReq; +import org.apache.seatunnel.app.domain.request.job.PluginConfig; +import org.apache.seatunnel.app.domain.request.job.TableSchemaReq; +import org.apache.seatunnel.app.domain.request.job.transform.SQL; +import org.apache.seatunnel.app.service.IJobTaskService; +import org.apache.seatunnel.app.service.ISchemaDerivationService; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; +import org.apache.seatunnel.transform.sql.SQLTransform; + +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Service +public class SchemaDerivationServiceImpl implements ISchemaDerivationService { + + @Resource private IJobTaskService jobTaskService; + + private static final Pattern decimalPattern = Pattern.compile("DECIMAL\\((\\d+), (\\d+)\\)"); + + @Override + public TableSchemaReq derivationSQL(long jobVersionId, String inputPluginId, SQL sql) { + + PluginConfig pluginConfig = jobTaskService.getSingleTask(jobVersionId, inputPluginId); + TableTransformFactory factory = + FactoryUtil.discoverFactory( + Thread.currentThread().getContextClassLoader(), + TableTransformFactory.class, + "Sql"); + List tableSchemaReqs = pluginConfig.getOutputSchema(); + if (tableSchemaReqs.isEmpty()) { + throw new IllegalArgumentException("outputSchema is empty, please add input plugin"); + } + DatabaseTableSchemaReq tableSchema = tableSchemaReqs.get(0); + TableSchema.Builder builder = TableSchema.builder(); + List primaryKeys = new ArrayList<>(); + for (TableField f : tableSchema.getFields()) { + if (f.getPrimaryKey()) { + primaryKeys.add(f.getName()); + } + builder.column( + PhysicalColumn.of( + f.getName(), + stringToDataType(f.getOutputDataType()), + 0, + f.getNullable(), + f.getDefaultValue(), + f.getComment())); + } + builder.primaryKey(PrimaryKey.of("PrimaryKeys", primaryKeys)); + + CatalogTable table = + CatalogTable.of( + TableIdentifier.of( + "default", tableSchema.getDatabase(), tableSchema.getTableName()), + builder.build(), + Collections.emptyMap(), + Collections.emptyList(), + tableSchema.getTableName()); + Map config = new HashMap<>(); + config.put(SQLTransform.KEY_QUERY.key(), sql.getQuery()); + TableFactoryContext context = + new TableFactoryContext( + Collections.singletonList(table), + ReadonlyConfig.fromMap(config), + Thread.currentThread().getContextClassLoader()); + TableTransform transform = factory.createTransform(context); + SQLTransform sqlTransform = (SQLTransform) transform.createTransform(); + CatalogTable result = sqlTransform.getProducedCatalogTable(); + List primaryKeysList = new ArrayList<>(); + if (result.getTableSchema().getPrimaryKey() != null) { + primaryKeysList.addAll(result.getTableSchema().getPrimaryKey().getColumnNames()); + } + List fields = new ArrayList<>(); + for (Column column : result.getTableSchema().getColumns()) { + TableField field = new TableField(); + field.setName(column.getName()); + field.setComment(column.getComment()); + field.setDefaultValue( + column.getDefaultValue() != null ? column.getDefaultValue().toString() : null); + field.setNullable(column.isNullable()); + field.setOutputDataType(column.getDataType().toString()); + field.setPrimaryKey(primaryKeysList.contains(column.getName())); + field.setType(column.getDataType().toString()); + fields.add(field); + } + + TableSchemaReq tableSchemaRes = new TableSchemaReq(); + tableSchemaRes.setFields(fields); + tableSchemaRes.setTableName(tableSchema.getTableName()); + return tableSchemaRes; + } + + private SeaTunnelDataType stringToDataType(String dataTypeStr) { + dataTypeStr = dataTypeStr.toUpperCase(); + switch (dataTypeStr) { + case "STRING": + return BasicType.STRING_TYPE; + case "BOOLEAN": + return BasicType.BOOLEAN_TYPE; + case "TINYINT": + return BasicType.BYTE_TYPE; + case "SMALLINT": + return BasicType.SHORT_TYPE; + case "INT": + return BasicType.INT_TYPE; + case "BIGINT": + return BasicType.LONG_TYPE; + case "FLOAT": + return BasicType.FLOAT_TYPE; + case "DOUBLE": + return BasicType.DOUBLE_TYPE; + case "NULL": + return BasicType.VOID_TYPE; + case "BYTES": + return ArrayType.BYTE_ARRAY_TYPE; + case "DATE": + return LocalTimeType.LOCAL_DATE_TYPE; + case "TIME": + return LocalTimeType.LOCAL_TIME_TYPE; + case "TIMESTAMP": + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case "DECIMAL": + return new DecimalType(38, 18); + case "ARRAY": + case "MAP": + case "ROW": + case "MULTIPLE_ROW": + return BasicType.STRING_TYPE; + default: + break; + } + + Matcher matcher = decimalPattern.matcher(dataTypeStr); + if (matcher.matches()) { + int precision = Integer.parseInt(matcher.group(1)); + int scale = Integer.parseInt(matcher.group(2)); + return new DecimalType(precision, scale); + } + return BasicType.STRING_TYPE; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TableSchemaServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TableSchemaServiceImpl.java index fd3ec8061..52c8c4e91 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TableSchemaServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TableSchemaServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.seatunnel.api.table.factory.DataTypeConvertorFactory; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.app.bean.connector.ConnectorCache; +import org.apache.seatunnel.app.bean.engine.EngineDataType; import org.apache.seatunnel.app.config.ConnectorDataSourceMapperConfig; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.TableSchemaReq; @@ -92,7 +93,16 @@ public TableSchemaRes getSeaTunnelSchema(String pluginName, TableSchemaReq table } else if (pluginName.startsWith("JDBC-")) { pluginName = pluginName.replace("JDBC-", ""); } - DataTypeConvertor convertor = factory.getDataTypeConvertor(pluginName); + // if the convertor is not exist in the plugin, will use the input type as the output type + DataTypeConvertor convertor; + try { + convertor = factory.getDataTypeConvertor(pluginName); + } catch (Exception e) { + convertor = new EngineDataType.SeaTunnelDataTypeConvertor(); + log.warn( + "The convertor of plugin: {} is not exist, will use EngineDataType.SeaTunnelDataTypeConvertor", + pluginName); + } for (TableField field : tableSchemaReq.getFields()) { SeaTunnelDataType dataType = convertor.toSeaTunnelType(field.getType()); @@ -113,7 +123,16 @@ public void getAddSeaTunnelSchema(List tableFields, String pluginNam } else if (pluginName.startsWith("JDBC-")) { pluginName = pluginName.replace("JDBC-", ""); } - DataTypeConvertor convertor = factory.getDataTypeConvertor(pluginName); + // if the convertor is not exist in the plugin, will use the input type as the output type + DataTypeConvertor convertor; + try { + convertor = factory.getDataTypeConvertor(pluginName); + } catch (Exception e) { + convertor = new EngineDataType.SeaTunnelDataTypeConvertor(); + log.warn( + "The convertor of plugin: {} is not exist, will use EngineDataType.SeaTunnelDataTypeConvertor", + pluginName); + } for (TableField field : tableFields) { try { SeaTunnelDataType dataType = convertor.toSeaTunnelType(field.getType()); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TaskInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TaskInstanceServiceImpl.java index 4515a53ae..aa89c85c3 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TaskInstanceServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/TaskInstanceServiceImpl.java @@ -30,10 +30,11 @@ import org.apache.seatunnel.app.service.ITaskInstanceService; import org.apache.seatunnel.app.utils.PageInfo; +import org.apache.commons.collections4.CollectionUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import cn.hutool.core.collection.CollUtil; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import lombok.extern.slf4j.Slf4j; @@ -47,7 +48,7 @@ @Service @Slf4j -public class TaskInstanceServiceImpl implements ITaskInstanceService { +public class TaskInstanceServiceImpl implements ITaskInstanceService { @Autowired IJobInstanceDao jobInstanceDao; @@ -60,7 +61,7 @@ public class TaskInstanceServiceImpl implements ITaskInstanceService { @Autowired IJobDefinitionDao jobDefinitionDao; @Override - public Result getSyncTaskInstancePaging( + public Result> getSyncTaskInstancePaging( Integer userId, String jobDefineName, String executorName, @@ -71,7 +72,7 @@ public Result getSyncTaskInstancePaging( Integer pageNo, Integer pageSize) { JobDefinition jobDefinition = null; - IPage jobInstanceIPage = null; + IPage jobInstanceIPage; if (jobDefineName != null) { jobDefinition = jobDefinitionDao.getJobByName(jobDefineName); } @@ -99,7 +100,7 @@ public Result getSyncTaskInstancePaging( } List records = jobInstanceIPage.getRecords(); - if (CollUtil.isEmpty(records)) { + if (CollectionUtils.isEmpty(records)) { return result; } addJobDefineNameToResult(records); @@ -144,8 +145,7 @@ public Date dateConverter(String time) { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); try { - Date date = dateFormat.parse(time); - return date; + return dateFormat.parse(time); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/VirtualTableServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/VirtualTableServiceImpl.java index 8a9c4f39b..50303c459 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/VirtualTableServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/VirtualTableServiceImpl.java @@ -154,7 +154,7 @@ public Boolean updateVirtualTable( VirtualTable virtualTable = VirtualTable.builder() .id(Long.valueOf(tableId)) - .datasourceId(originalTable.getDatasourceId()) + .datasourceId(Long.parseLong(req.getDatasourceId())) .virtualDatabaseName(req.getDatabaseName()) .virtualTableName(req.getTableName()) .description(req.getDescription()) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/AbstractDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/AbstractDataSourceConfigSwitcher.java index 975145f1e..ccf0f8062 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/AbstractDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/AbstractDataSourceConfigSwitcher.java @@ -21,6 +21,8 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.util.Condition; +import org.apache.seatunnel.api.configuration.util.Expression; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; @@ -47,6 +49,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { List dataSourceRequiredAllKey = @@ -58,6 +62,15 @@ public FormStructure filterOptionRule( dataSourceRequiredAllKey.addAll(excludedKeys); + List dataSourceOptionAllKey = + Stream.concat( + dataSourceOptionRule.getOptionalOptions().stream(), + virtualTableOptionRule.getOptionalOptions().stream()) + .map(Option::key) + .collect(Collectors.toList()); + + dataSourceOptionAllKey.addAll(excludedKeys); + List requiredOptions = connectorOptionRule.getRequiredOptions().stream() .map( @@ -73,7 +86,13 @@ public FormStructure filterOptionRule( .filter( op -> { return !dataSourceRequiredAllKey - .contains(op.key()); + .contains( + op + .key()) + & !dataSourceOptionAllKey + .contains( + op + .key()); }) .collect(Collectors.toList()); return requiredOpList.isEmpty() @@ -117,13 +136,31 @@ public FormStructure filterOptionRule( instanceof RequiredOption.ConditionalRequiredOptions) { List> conditionalRequiredOptions = requiredOption.getOptions(); - return conditionalRequiredOptions.stream() - .anyMatch( - op -> - dataSourceRequiredAllKey - .contains(op.key())) - ? null - : requiredOption; + RequiredOption.ConditionalRequiredOptions + conditionalRequired = + (RequiredOption.ConditionalRequiredOptions) + requiredOption; + Expression expression = conditionalRequired.getExpression(); + Condition condition = expression.getCondition(); + List> requiredOptionList = + conditionalRequired.getRequiredOption(); + List> requiredOpList = + requiredOptionList.stream() + .filter( + op -> { + return !dataSourceRequiredAllKey + .contains( + op + .key()) + & !dataSourceOptionAllKey + .contains( + op + .key()); + }) + .collect(Collectors.toList()); + Expression expressionNew = Expression.of(condition); + return RequiredOption.ConditionalRequiredOptions.of( + expressionNew, requiredOpList); } throw new UnSupportWrapperException( @@ -132,20 +169,13 @@ public FormStructure filterOptionRule( .filter(Objects::nonNull) .collect(Collectors.toList()); - List dataSourceOptionAllKey = - Stream.concat( - dataSourceOptionRule.getOptionalOptions().stream(), - virtualTableOptionRule.getOptionalOptions().stream()) - .map(Option::key) - .collect(Collectors.toList()); - - dataSourceOptionAllKey.addAll(excludedKeys); - List> optionList = connectorOptionRule.getOptionalOptions().stream() .filter(option -> !dataSourceOptionAllKey.contains(option.key())) .collect(Collectors.toList()); + requiredOptions.addAll(addRequiredOptions); + optionList.addAll(addOptionalOptions); return SeaTunnelOptionRuleWrapper.wrapper( optionList, requiredOptions, connectorName, pluginType); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcher.java index 443d09147..d9a4acfaa 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcher.java @@ -19,7 +19,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -31,6 +33,8 @@ public interface DataSourceConfigSwitcher { + String getDataSourceName(); + /** * OptionRule OptionRule * @@ -50,6 +54,8 @@ FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys); /** diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcherUtils.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcherUtils.java index b82b390f2..8147fea6c 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcherUtils.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DataSourceConfigSwitcherUtils.java @@ -25,25 +25,6 @@ import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; -import org.apache.seatunnel.app.thirdparty.datasource.impl.ClickhouseDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.DamengDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.ElasticSearchDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.KafkaDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.KafkaKingbaseDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.KingBaseDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.MysqlCDCDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.MysqlDatasourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.OracleCDCDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.OracleDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.PostgresCDCDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.PostgresqlDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.RedshiftDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.S3DataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.S3RedshiftDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.SqlServerCDCDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.SqlServerDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.StarRocksDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.impl.TidbDataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.common.utils.SeaTunnelException; @@ -70,6 +51,8 @@ public static FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + new ArrayList<>(), + new ArrayList<>(), new ArrayList<>()); } @@ -96,52 +79,15 @@ public static Config mergeDatasourceConfig( private static DataSourceConfigSwitcher getDataSourceConfigSwitcher(String datasourceName) { checkNotNull(datasourceName, "datasourceName cannot be null"); - // Use SPI - switch (datasourceName.toUpperCase()) { - case "JDBC-MYSQL": - return MysqlDatasourceConfigSwitcher.INSTANCE; - case "ELASTICSEARCH": - return ElasticSearchDataSourceConfigSwitcher.INSTANCE; - case "KAFKA": - return KafkaDataSourceConfigSwitcher.getInstance(); - case "STARROCKS": - return StarRocksDataSourceConfigSwitcher.INSTANCE; - case "MYSQL-CDC": - return MysqlCDCDataSourceConfigSwitcher.INSTANCE; - case "S3-REDSHIFT": - return S3RedshiftDataSourceConfigSwitcher.getInstance(); - case "JDBC-CLICKHOUSE": - return ClickhouseDataSourceConfigSwitcher.getInstance(); - case "JDBC-DAMENG": - return DamengDataSourceConfigSwitcher.getInstance(); - case "JDBC-POSTGRES": - return PostgresqlDataSourceConfigSwitcher.getInstance(); - case "JDBC-REDSHIFT": - return RedshiftDataSourceConfigSwitcher.getInstance(); - case "JDBC-SQLSERVER": - return SqlServerDataSourceConfigSwitcher.getInstance(); - case "JDBC-TIDB": - return TidbDataSourceConfigSwitcher.INSTANCE; - case "JDBC-ORACLE": - return OracleDataSourceConfigSwitcher.INSTANCE; - case "S3": - return S3DataSourceConfigSwitcher.getInstance(); - case "SQLSERVER-CDC": - return SqlServerCDCDataSourceConfigSwitcher.INSTANCE; - case "POSTGRES-CDC": - return PostgresCDCDataSourceConfigSwitcher.INSTANCE; - case "JDBC-KINGBASE": - return KingBaseDataSourceConfigSwitcher.getInstance(); - case "ORACLE-CDC": - return OracleCDCDataSourceConfigSwitcher.INSTANCE; - case "KAFKA-KINGBASE": - return KafkaKingbaseDataSourceConfigSwitcher.getInstance(); - - default: - throw new SeaTunnelException( - "data source : " - + datasourceName - + " is no implementation class for DataSourceConfigSwitcher"); + DataSourceConfigSwitcher configSwitcher = + DatasourceConfigSwitcherProvider.INSTANCE.getConfigSwitcher( + datasourceName.toUpperCase()); + if (configSwitcher == null) { + throw new SeaTunnelException( + "data source : " + + datasourceName + + " is no implementation class for DataSourceConfigSwitcher"); } + return configSwitcher; } } diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/OnlineCreateResourceDto.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DatasourceConfigSwitcherProvider.java similarity index 51% rename from seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/OnlineCreateResourceDto.java rename to seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DatasourceConfigSwitcherProvider.java index acd909b73..795b07a84 100644 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/OnlineCreateResourceDto.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/DatasourceConfigSwitcherProvider.java @@ -15,32 +15,28 @@ * limitations under the License. */ -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.google.common.collect.Maps; -import lombok.Builder; -import lombok.Data; +package org.apache.seatunnel.app.thirdparty.datasource; import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +public enum DatasourceConfigSwitcherProvider { + INSTANCE; + + private final Map configSwitcherCache; -@Builder -@Data -public class OnlineCreateResourceDto { - private String type; - private int pid; - private String currentDir; - private String fileName; - private String suffix; - private String content; + DatasourceConfigSwitcherProvider() { + ServiceLoader loader = + ServiceLoader.load(DataSourceConfigSwitcher.class); + configSwitcherCache = new ConcurrentHashMap<>(); + + for (DataSourceConfigSwitcher switcher : loader) { + configSwitcherCache.put(switcher.getDataSourceName().toUpperCase(), switcher); + } + } - public Map toMap() { - final Map map = Maps.newHashMap(); - map.put("type", type); - map.put("pid", pid); - map.put("currentDir", currentDir); - map.put("fileName", fileName); - map.put("suffix", suffix); - map.put("content", content); - return map; + public DataSourceConfigSwitcher getConfigSwitcher(String datasourceName) { + return configSwitcherCache.get(datasourceName); } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java index 150ed95f7..d3cdb8e8b 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java @@ -18,9 +18,13 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -34,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; @@ -48,6 +53,20 @@ public abstract class BaseJdbcDataSourceConfigSwitcher extends AbstractDataSourc private static final String URL_KEY = "url"; + // for catalog + private static final String CATALOG = "catalog"; + private static final String FACTORY = "factory"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + private static final String BASE_URL = "base-url"; + private static final String CATALOG_SCHEMA = "schema"; + + private static final Option DATABASE_SCHEMA = + Options.key("database_schema") + .stringType() + .noDefaultValue() + .withDescription("the default database used during automated table creation."); + @Override public FormStructure filterOptionRule( String connectorName, @@ -56,6 +75,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { Map> filterFieldMap = new HashMap<>(); @@ -63,7 +84,11 @@ public FormStructure filterOptionRule( PluginType.SINK, Arrays.asList(QUERY_KEY, TABLE_KEY, DATABASE_KEY, GENERATE_SINK_SQL)); filterFieldMap.put(PluginType.SOURCE, Collections.singletonList(QUERY_KEY)); - + if (isSupportDefaultSchema() + && businessMode.equals(DATA_REPLICA) + && pluginType.equals(PluginType.SINK)) { + addOptionalOptions.add(DATABASE_SCHEMA); + } return super.filterOptionRule( connectorName, dataSourceOptionRule, @@ -71,6 +96,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, filterFieldMap.get(pluginType)); } @@ -135,6 +162,37 @@ public Config mergeDatasourceConfig( } else if (businessMode.equals(DATA_REPLICA)) { String databaseName = dataSourceOption.getDatabases().get(0); if (pluginType.equals(PluginType.SINK)) { + if (getCatalogName().isPresent()) { + Config config = ConfigFactory.empty(); + config = + config.withValue( + FACTORY, ConfigValueFactory.fromAnyRef(getCatalogName().get())); + config = + config.withValue( + USERNAME, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString("user"))); + config = + config.withValue( + PASSWORD, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString(PASSWORD))); + config = + config.withValue( + BASE_URL, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString(URL_KEY))); + if (isSupportDefaultSchema() + && connectorConfig.hasPath(DATABASE_SCHEMA.key())) { + config = + config.withValue( + CATALOG_SCHEMA, + ConfigValueFactory.fromAnyRef( + connectorConfig.getString(DATABASE_SCHEMA.key()))); + } + + connectorConfig = connectorConfig.withValue(CATALOG, config.root()); + } connectorConfig = connectorConfig.withValue( DATABASE_KEY, ConfigValueFactory.fromAnyRef(databaseName)); @@ -175,6 +233,14 @@ protected String generateSql( return sb.toString(); } + protected boolean isSupportDefaultSchema() { + return false; + } + + protected Optional getCatalogName() { + return Optional.empty(); + } + protected String tableFieldsToSql(List tableFields, String database, String table) { return generateSql(tableFields, database, null, table); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ClickhouseDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ClickhouseDataSourceConfigSwitcher.java index 4e401d2be..3c453f3aa 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ClickhouseDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ClickhouseDataSourceConfigSwitcher.java @@ -20,25 +20,28 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.app.utils.JdbcUtils; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.util.List; import java.util.Map; +@AutoService(DataSourceConfigSwitcher.class) public class ClickhouseDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private static final ClickhouseDataSourceConfigSwitcher INSTANCE = - new ClickhouseDataSourceConfigSwitcher(); private static final String HOST = "host"; private static final String URL = "url"; @@ -52,6 +55,11 @@ public class ClickhouseDataSourceConfigSwitcher extends AbstractDataSourceConfig .put(PluginType.SINK, Lists.newArrayList(HOST, DATABASE, TABLE)) .build(); + @Override + public String getDataSourceName() { + return "JDBC-CLICKHOUSE"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -60,6 +68,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { return super.filterOptionRule( connectorName, @@ -68,6 +78,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, FILTER_FIELD_MAP.get(pluginType)); } @@ -143,9 +155,5 @@ public Config mergeDatasourceConfig( connectorConfig); } - public static ClickhouseDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } - - private ClickhouseDataSourceConfigSwitcher() {} + public ClickhouseDataSourceConfigSwitcher() {} } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DamengDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DamengDataSourceConfigSwitcher.java index a5681601f..60ef1f0a4 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DamengDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DamengDataSourceConfigSwitcher.java @@ -17,13 +17,17 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceConfigSwitcher.class) public class DamengDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - private static final DamengDataSourceConfigSwitcher INSTANCE = - new DamengDataSourceConfigSwitcher(); - public static final DamengDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } + public DamengDataSourceConfigSwitcher() {} - private DamengDataSourceConfigSwitcher() {} + @Override + public String getDataSourceName() { + return "JDBC-DAMENG"; + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/Db2DataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/Db2DataSourceConfigSwitcher.java new file mode 100644 index 000000000..327f98290 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/Db2DataSourceConfigSwitcher.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.thirdparty.datasource.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.app.domain.request.connector.BusinessMode; +import org.apache.seatunnel.app.domain.request.job.DataSourceOption; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.common.constants.PluginType; + +import com.google.auto.service.AutoService; + +import java.util.List; +import java.util.Optional; + +import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; +import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; + +@AutoService(DataSourceConfigSwitcher.class) +public class Db2DataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { + + private static final String QUERY_KEY = "query"; + + private static final String GENERATE_SINK_SQL = "generate_sink_sql"; + + private static final String URL_KEY = "url"; + + // for catalog + private static final String CATALOG = "catalog"; + private static final String FACTORY = "factory"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + private static final String BASE_URL = "base-url"; + private static final String CATALOG_SCHEMA = "schema"; + + private static final Option DATABASE_SCHEMA = + Options.key("database_schema") + .stringType() + .noDefaultValue() + .withDescription("the default database used during automated table creation."); + private static final String CATALOG_NAME = "Db2"; + + public Db2DataSourceConfigSwitcher() {} + + protected Optional getCatalogName() { + return Optional.of(CATALOG_NAME); + } + + protected boolean isSupportPrefixOrSuffix() { + return true; + } + + protected boolean isSupportToggleCase() { + return true; + } + + public Config mergeDatasourceConfig( + Config dataSourceInstanceConfig, + VirtualTableDetailRes virtualTableDetail, + DataSourceOption dataSourceOption, + SelectTableFields selectTableFields, + BusinessMode businessMode, + PluginType pluginType, + Config connectorConfig) { + + // 替换url中的database + if (dataSourceOption.getDatabases().size() == 1) { + String databaseName = dataSourceOption.getDatabases().get(0); + String url = dataSourceInstanceConfig.getString(URL_KEY); + String newUrl = replaceDatabaseNameInUrl(url, databaseName); + dataSourceInstanceConfig = + dataSourceInstanceConfig.withValue( + URL_KEY, ConfigValueFactory.fromAnyRef(newUrl)); + } + if (businessMode.equals(DATA_INTEGRATION)) { + + String databaseName = dataSourceOption.getDatabases().get(0); + + String tableName = dataSourceOption.getTables().get(0); + + // 将schema转换成sql + if (pluginType.equals(PluginType.SOURCE)) { + + List tableFields = selectTableFields.getTableFields(); + + String sql = tableFieldsToSql(tableFields, databaseName, tableName); + + connectorConfig = + connectorConfig.withValue(QUERY_KEY, ConfigValueFactory.fromAnyRef(sql)); + } else if (pluginType.equals(PluginType.SINK)) { + + List tableFields = selectTableFields.getTableFields(); + + String sql = generateDb2(tableFields, databaseName, tableName); + + connectorConfig = + connectorConfig.withValue(QUERY_KEY, ConfigValueFactory.fromAnyRef(sql)); + } else { + throw new UnsupportedOperationException("Unsupported plugin type: " + pluginType); + } + + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } else if (businessMode.equals(DATA_REPLICA)) { + String databaseName = dataSourceOption.getDatabases().get(0); + if (pluginType.equals(PluginType.SINK)) { + if (getCatalogName().isPresent()) { + Config config = ConfigFactory.empty(); + config = + config.withValue( + FACTORY, ConfigValueFactory.fromAnyRef(getCatalogName().get())); + config = + config.withValue( + USERNAME, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString("user"))); + config = + config.withValue( + PASSWORD, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString(PASSWORD))); + config = + config.withValue( + BASE_URL, + ConfigValueFactory.fromAnyRef( + dataSourceInstanceConfig.getString(URL_KEY))); + if (isSupportDefaultSchema() + && connectorConfig.hasPath(DATABASE_SCHEMA.key())) { + config = + config.withValue( + CATALOG_SCHEMA, + ConfigValueFactory.fromAnyRef( + connectorConfig.getString(DATABASE_SCHEMA.key()))); + } + + connectorConfig = connectorConfig.withValue(CATALOG, config.root()); + } + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } else { + throw new UnsupportedOperationException( + "JDBC DATA_REPLICA Unsupported plugin type: " + pluginType); + } + + } else { + throw new UnsupportedOperationException("Unsupported businessMode : " + businessMode); + } + } + + protected String tableFieldsToSql(List tableFields, String database, String table) { + return generateSql(tableFields, database, null, table); + } + + protected String generateDb2(List tableFields, String database, String table) { + return generateSinkSql(tableFields, database, null, table); + } + + protected String generateSql( + List tableFields, String database, String schema, String table) { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + for (int i = 0; i < tableFields.size(); i++) { + sb.append(quoteIdentifier(tableFields.get(i))); + if (i < tableFields.size() - 1) { + sb.append(", "); + } + } + sb.append(" FROM ").append(quoteIdentifier(table)); + + return sb.toString(); + } + + protected String quoteIdentifier(String identifier) { + return "\"" + identifier + "\""; + } + + protected String generateSinkSql( + List tableFields, String database, String schema, String table) { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO ").append(quoteIdentifier(table)).append(" ("); + + // Append column names + for (int i = 0; i < tableFields.size(); i++) { + sb.append(quoteIdentifier(tableFields.get(i))); + if (i < tableFields.size() - 1) { + sb.append(", "); + } + } + + sb.append(") VALUES ("); + + // Append placeholders + for (int i = 0; i < tableFields.size(); i++) { + sb.append("?"); + if (i < tableFields.size() - 1) { + sb.append(", "); + } + } + + sb.append(");"); + return sb.toString(); + } + + @Override + public String getDataSourceName() { + return "JDBC-DB2"; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ElasticSearchDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ElasticSearchDataSourceConfigSwitcher.java index feb3304d8..ac7631ee3 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ElasticSearchDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/ElasticSearchDataSourceConfigSwitcher.java @@ -20,29 +20,37 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.Arrays; import java.util.List; +@AutoService(DataSourceConfigSwitcher.class) public class ElasticSearchDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - public static final ElasticSearchDataSourceConfigSwitcher INSTANCE = - new ElasticSearchDataSourceConfigSwitcher(); - private static final String SOURCE = "source"; private static final String SCHEMA = "schema"; private static final String PRIMARY_KEYS = "primary_keys"; private static final String INDEX = "index"; - private ElasticSearchDataSourceConfigSwitcher() {} + public ElasticSearchDataSourceConfigSwitcher() {} + + @Override + public String getDataSourceName() { + return "ELASTICSEARCH"; + } @Override public FormStructure filterOptionRule( @@ -52,6 +60,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { // DELETE source/schema @@ -72,6 +82,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaDataSourceConfigSwitcher.java index ece872579..a175396e1 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaDataSourceConfigSwitcher.java @@ -20,22 +20,25 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.List; +@AutoService(DataSourceConfigSwitcher.class) public class KafkaDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private static final KafkaDataSourceConfigSwitcher INSTANCE = - new KafkaDataSourceConfigSwitcher(); - private static final String SCHEMA = "schema"; private static final String TOPIC = "topic"; private static final String TABLE = "table"; @@ -43,6 +46,11 @@ public class KafkaDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitc private static final String DEBEZIUM_FORMAT = "COMPATIBLE_DEBEZIUM_JSON"; + @Override + public String getDataSourceName() { + return "KAFKA"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -51,6 +59,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (pluginType == PluginType.SOURCE) { excludedKeys.add(SCHEMA); @@ -66,6 +76,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } @@ -120,9 +132,5 @@ public Config mergeDatasourceConfig( connectorConfig); } - private KafkaDataSourceConfigSwitcher() {} - - public static KafkaDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } + public KafkaDataSourceConfigSwitcher() {} } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaKingbaseDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaKingbaseDataSourceConfigSwitcher.java index 1c83d763a..06462fa5d 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaKingbaseDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KafkaKingbaseDataSourceConfigSwitcher.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -29,11 +31,13 @@ import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -45,11 +49,9 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +@AutoService(DataSourceConfigSwitcher.class) public class KafkaKingbaseDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private static final KafkaKingbaseDataSourceConfigSwitcher INSTANCE = - new KafkaKingbaseDataSourceConfigSwitcher(); - private static final String SCHEMA = "schema"; private static final String TOPIC = "topic"; private static final String FORMAT = "format"; @@ -69,6 +71,11 @@ public class KafkaKingbaseDataSourceConfigSwitcher extends AbstractDataSourceCon private static final String DATABASE_NAMES = "database-names"; + @Override + public String getDataSourceName() { + return "KAFKA-KINGBASE"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -77,6 +84,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (pluginType == PluginType.SOURCE) { excludedKeys.add(SCHEMA); @@ -93,6 +102,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } @@ -168,11 +179,7 @@ private static String getDatabaseAndTable(String database, String table) { return String.format("%s.%s", database, table); } - private KafkaKingbaseDataSourceConfigSwitcher() {} - - public static KafkaKingbaseDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } + public KafkaKingbaseDataSourceConfigSwitcher() {} /* * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KingBaseDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KingBaseDataSourceConfigSwitcher.java deleted file mode 100644 index 4d5ad7d8e..000000000 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/KingBaseDataSourceConfigSwitcher.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.app.thirdparty.datasource.impl; - -import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; -import org.apache.seatunnel.common.utils.SeaTunnelException; - -import java.util.List; - -public class KingBaseDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - - private static final KingBaseDataSourceConfigSwitcher INSTANCE = - new KingBaseDataSourceConfigSwitcher(); - - private KingBaseDataSourceConfigSwitcher() {} - - protected String tableFieldsToSql(List tableFields, String database, String fullTable) { - - String[] split = fullTable.split("\\."); - if (split.length != 2) { - throw new SeaTunnelException( - "The tableName for postgres must be schemaName.tableName, but tableName is " - + fullTable); - } - - String schemaName = split[0]; - String tableName = split[1]; - - return generateSql(tableFields, database, schemaName, tableName); - } - - protected String quoteIdentifier(String identifier) { - return "\"" + identifier + "\""; - } - - public static DataSourceConfigSwitcher getInstance() { - return (DataSourceConfigSwitcher) INSTANCE; - } -} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java new file mode 100644 index 000000000..2153c52e7 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.thirdparty.datasource.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; +import org.apache.seatunnel.app.domain.request.connector.BusinessMode; +import org.apache.seatunnel.app.domain.request.job.DataSourceOption; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.dynamicforms.FormStructure; +import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.common.constants.PluginType; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@AutoService(DataSourceConfigSwitcher.class) +@Slf4j +public class MongoDBDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { + private static final String DATABASE = "database"; + private static final String COLLECTION = "collection"; + private static final String SCHEMA = "schema"; + + @Override + public String getDataSourceName() { + return "MONGODB"; + } + + @Override + public FormStructure filterOptionRule( + String connectorName, + OptionRule dataSourceOptionRule, + OptionRule virtualTableOptionRule, + BusinessMode businessMode, + PluginType pluginType, + OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, + List excludedKeys) { + excludedKeys.add(SCHEMA); + return super.filterOptionRule( + connectorName, + dataSourceOptionRule, + virtualTableOptionRule, + businessMode, + pluginType, + connectorOptionRule, + addRequiredOptions, + addOptionalOptions, + excludedKeys); + } + + @Override + public Config mergeDatasourceConfig( + Config dataSourceInstanceConfig, + VirtualTableDetailRes virtualTableDetail, + DataSourceOption dataSourceOption, + SelectTableFields selectTableFields, + BusinessMode businessMode, + PluginType pluginType, + Config connectorConfig) { + // Use field to generate the schema + connectorConfig = + connectorConfig.withValue( + DATABASE, + ConfigValueFactory.fromAnyRef( + virtualTableDetail.getDatasourceProperties().get(DATABASE))); + connectorConfig = + connectorConfig.withValue( + COLLECTION, + ConfigValueFactory.fromAnyRef( + virtualTableDetail.getDatasourceProperties().get(COLLECTION))); + if (pluginType == PluginType.SOURCE) { + connectorConfig = + connectorConfig.withValue( + SCHEMA, + KafkaKingbaseDataSourceConfigSwitcher.SchemaGenerator + .generateSchemaBySelectTableFields( + virtualTableDetail, selectTableFields) + .root()); + } + + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlCDCDataSourceConfigSwitcher.java index 8045f8d9d..528e73c7c 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlCDCDataSourceConfigSwitcher.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -29,8 +31,11 @@ import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -38,12 +43,10 @@ import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; +@AutoService(DataSourceConfigSwitcher.class) public class MysqlCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private MysqlCDCDataSourceConfigSwitcher() {} - - public static final MysqlCDCDataSourceConfigSwitcher INSTANCE = - new MysqlCDCDataSourceConfigSwitcher(); + public MysqlCDCDataSourceConfigSwitcher() {} private static final String FACTORY = "factory"; @@ -61,6 +64,11 @@ private MysqlCDCDataSourceConfigSwitcher() {} private static final String SCHEMA = "schema"; + @Override + public String getDataSourceName() { + return "MYSQL-CDC"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -69,6 +77,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { excludedKeys.add(DATABASE_NAMES); @@ -86,6 +96,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlDatasourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlDatasourceConfigSwitcher.java index a034a6430..7bd94c2c5 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlDatasourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MysqlDatasourceConfigSwitcher.java @@ -17,13 +17,17 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; + +import com.google.auto.service.AutoService; + import java.util.Optional; +@AutoService(DataSourceConfigSwitcher.class) public class MysqlDatasourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - public static MysqlDatasourceConfigSwitcher INSTANCE = new MysqlDatasourceConfigSwitcher(); private static final String CATALOG_NAME = "MySQL"; - private MysqlDatasourceConfigSwitcher() {} + public MysqlDatasourceConfigSwitcher() {} protected Optional getCatalogName() { return Optional.of(CATALOG_NAME); @@ -36,4 +40,9 @@ protected boolean isSupportPrefixOrSuffix() { protected boolean isSupportToggleCase() { return true; } + + @Override + public String getDataSourceName() { + return "JDBC-MYSQL"; + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleCDCDataSourceConfigSwitcher.java index 093df5e94..879bfa679 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleCDCDataSourceConfigSwitcher.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -29,8 +31,11 @@ import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -38,12 +43,10 @@ import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; +@AutoService(DataSourceConfigSwitcher.class) public class OracleCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private OracleCDCDataSourceConfigSwitcher() {} - - public static final OracleCDCDataSourceConfigSwitcher INSTANCE = - new OracleCDCDataSourceConfigSwitcher(); + public OracleCDCDataSourceConfigSwitcher() {} private static final String FACTORY = "factory"; @@ -61,6 +64,11 @@ private OracleCDCDataSourceConfigSwitcher() {} private static final String SCHEMA = "schema"; + @Override + public String getDataSourceName() { + return "ORACLE-CDC"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -69,6 +77,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { excludedKeys.add(DATABASE_NAMES); @@ -86,6 +96,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleDataSourceConfigSwitcher.java index 58640ae76..fbcda2e6e 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/OracleDataSourceConfigSwitcher.java @@ -17,17 +17,18 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.utils.SeaTunnelException; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; +import com.google.auto.service.AutoService; import java.util.List; +import java.util.Optional; -@NoArgsConstructor(access = AccessLevel.PRIVATE) +@AutoService(DataSourceConfigSwitcher.class) public class OracleDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - public static final OracleDataSourceConfigSwitcher INSTANCE = - new OracleDataSourceConfigSwitcher(); + + public static final String CATALOG_NAME = "Oracle"; protected String tableFieldsToSql(List tableFields, String database, String fullTable) { String[] split = fullTable.split("\\."); @@ -64,4 +65,17 @@ protected String generateSql( .append(quoteIdentifier(table)); return sb.toString(); } + + @Override + public String getDataSourceName() { + return "JDBC-ORACLE"; + } + + protected boolean isSupportDefaultSchema() { + return true; + } + + protected Optional getCatalogName() { + return Optional.of(CATALOG_NAME); + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresCDCDataSourceConfigSwitcher.java index 80e0947ba..18504ade4 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresCDCDataSourceConfigSwitcher.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -29,8 +31,11 @@ import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -38,15 +43,13 @@ import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; +@AutoService(DataSourceConfigSwitcher.class) public class PostgresCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { private int three = 3; private int two = 2; - private PostgresCDCDataSourceConfigSwitcher() {} - - public static final PostgresCDCDataSourceConfigSwitcher INSTANCE = - new PostgresCDCDataSourceConfigSwitcher(); + public PostgresCDCDataSourceConfigSwitcher() {} private static final String FACTORY = "factory"; @@ -64,6 +67,11 @@ private PostgresCDCDataSourceConfigSwitcher() {} private static final String SCHEMA = "schema"; + @Override + public String getDataSourceName() { + return "POSTGRES-CDC"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -72,6 +80,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { excludedKeys.add(DATABASE_NAMES); @@ -89,6 +99,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresqlDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresqlDataSourceConfigSwitcher.java index 94ca767c8..77982cad9 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresqlDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/PostgresqlDataSourceConfigSwitcher.java @@ -21,14 +21,21 @@ import org.apache.seatunnel.common.utils.SeaTunnelException; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; +import com.google.auto.service.AutoService; + import java.util.List; +import java.util.Optional; +@AutoService(DataSourceConfigSwitcher.class) public class PostgresqlDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - private static final PostgresqlDataSourceConfigSwitcher INSTANCE = - new PostgresqlDataSourceConfigSwitcher(); + public static final String CATALOG_NAME = "Postgres"; + + public PostgresqlDataSourceConfigSwitcher() {} - private PostgresqlDataSourceConfigSwitcher() {} + protected Optional getCatalogName() { + return Optional.of(CATALOG_NAME); + } protected String tableFieldsToSql(List tableFields, String database, String fullTable) { @@ -53,7 +60,12 @@ protected String replaceDatabaseNameInUrl(String url, String databaseName) { return JdbcUtils.replaceDatabase(url, databaseName); } - public static DataSourceConfigSwitcher getInstance() { - return (DataSourceConfigSwitcher) INSTANCE; + @Override + public String getDataSourceName() { + return "JDBC-POSTGRES"; + } + + protected boolean isSupportDefaultSchema() { + return true; } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/RedshiftDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/RedshiftDataSourceConfigSwitcher.java index 2752812e8..4f6c52cf2 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/RedshiftDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/RedshiftDataSourceConfigSwitcher.java @@ -17,18 +17,16 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.utils.SeaTunnelException; import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; +import com.google.auto.service.AutoService; + import java.util.List; +@AutoService(DataSourceConfigSwitcher.class) public class RedshiftDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - private static final RedshiftDataSourceConfigSwitcher INSTANCE = - new RedshiftDataSourceConfigSwitcher(); - - public static final RedshiftDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } protected String tableFieldsToSql(List tableFields, String database, String fullTable) { @@ -53,5 +51,10 @@ protected String replaceDatabaseNameInUrl(String url, String databaseName) { return JdbcUtils.replaceDatabase(url, databaseName); } - private RedshiftDataSourceConfigSwitcher() {} + public RedshiftDataSourceConfigSwitcher() {} + + @Override + public String getDataSourceName() { + return "JDBC-REDSHIFT"; + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3DataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3DataSourceConfigSwitcher.java index 814430ab2..00bfdbba6 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3DataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3DataSourceConfigSwitcher.java @@ -20,26 +20,34 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.datasource.plugin.s3.S3OptionRule; +import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import java.util.List; @Slf4j +@AutoService(DataSourceConfigSwitcher.class) public class S3DataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private S3DataSourceConfigSwitcher() {} + public S3DataSourceConfigSwitcher() {} - private static final S3DataSourceConfigSwitcher INSTANCE = new S3DataSourceConfigSwitcher(); + @Override + public String getDataSourceName() { + return "S3"; + } @Override public FormStructure filterOptionRule( @@ -49,6 +57,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { excludedKeys.add(S3OptionRule.PATH.key()); if (PluginType.SOURCE.equals(pluginType)) { @@ -62,6 +72,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } @@ -178,8 +190,4 @@ public Config mergeDatasourceConfig( pluginType, connectorConfig); } - - public static S3DataSourceConfigSwitcher getInstance() { - return INSTANCE; - } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3RedshiftDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3RedshiftDataSourceConfigSwitcher.java index a451031bd..795923fcc 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3RedshiftDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/S3RedshiftDataSourceConfigSwitcher.java @@ -19,23 +19,31 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.List; +@AutoService(DataSourceConfigSwitcher.class) public class S3RedshiftDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private S3RedshiftDataSourceConfigSwitcher() {} + public S3RedshiftDataSourceConfigSwitcher() {} - private static final S3RedshiftDataSourceConfigSwitcher INSTANCE = - new S3RedshiftDataSourceConfigSwitcher(); + @Override + public String getDataSourceName() { + return "S3-REDSHIFT"; + } @Override public FormStructure filterOptionRule( @@ -45,6 +53,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { excludedKeys.add("access_key"); excludedKeys.add("secret_key"); @@ -55,6 +65,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } @@ -76,8 +88,4 @@ public Config mergeDatasourceConfig( pluginType, connectorConfig); } - - public static S3RedshiftDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerCDCDataSourceConfigSwitcher.java index 9d246bf70..18cfd4235 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerCDCDataSourceConfigSwitcher.java @@ -21,7 +21,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; @@ -29,8 +31,11 @@ import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -38,12 +43,10 @@ import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_INTEGRATION; import static org.apache.seatunnel.app.domain.request.connector.BusinessMode.DATA_REPLICA; +@AutoService(DataSourceConfigSwitcher.class) public class SqlServerCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private SqlServerCDCDataSourceConfigSwitcher() {} - - public static final SqlServerCDCDataSourceConfigSwitcher INSTANCE = - new SqlServerCDCDataSourceConfigSwitcher(); + public SqlServerCDCDataSourceConfigSwitcher() {} private static final String FACTORY = "factory"; @@ -61,6 +64,11 @@ private SqlServerCDCDataSourceConfigSwitcher() {} private static final String SCHEMA = "schema"; + @Override + public String getDataSourceName() { + return "SQLSERVER-CDC"; + } + @Override public FormStructure filterOptionRule( String connectorName, @@ -69,6 +77,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { excludedKeys.add(DATABASE_NAMES); @@ -86,6 +96,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerDataSourceConfigSwitcher.java index ed816f14b..ad2fad53e 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/SqlServerDataSourceConfigSwitcher.java @@ -17,16 +17,17 @@ package org.apache.seatunnel.app.thirdparty.datasource.impl; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.utils.SeaTunnelException; +import com.google.auto.service.AutoService; + import java.util.List; import java.util.Optional; +@AutoService(DataSourceConfigSwitcher.class) public class SqlServerDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - private static final SqlServerDataSourceConfigSwitcher INSTANCE = - new SqlServerDataSourceConfigSwitcher(); - public static final String CATALOG_NAME = "SqlServer"; protected boolean isSupportDefaultSchema() { @@ -45,11 +46,7 @@ protected Optional getCatalogName() { return Optional.of(CATALOG_NAME); } - public static final SqlServerDataSourceConfigSwitcher getInstance() { - return INSTANCE; - } - - private SqlServerDataSourceConfigSwitcher() {} + public SqlServerDataSourceConfigSwitcher() {} protected String tableFieldsToSql(List tableFields, String database, String fullTable) { @@ -69,4 +66,9 @@ protected String tableFieldsToSql(List tableFields, String database, Str protected String quoteIdentifier(String identifier) { return "[" + identifier + "]"; } + + @Override + public String getDataSourceName() { + return "JDBC-SQLSERVER"; + } } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/StarRocksDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/StarRocksDataSourceConfigSwitcher.java index a6034d104..cb659293e 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/StarRocksDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/StarRocksDataSourceConfigSwitcher.java @@ -20,28 +20,36 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.api.configuration.Option; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.domain.request.connector.BusinessMode; import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + import java.util.Arrays; import java.util.List; +@AutoService(DataSourceConfigSwitcher.class) public class StarRocksDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { private static final String TABLE = "table"; private static final String DATABASE = "database"; - private StarRocksDataSourceConfigSwitcher() {} + public StarRocksDataSourceConfigSwitcher() {} - public static final StarRocksDataSourceConfigSwitcher INSTANCE = - new StarRocksDataSourceConfigSwitcher(); + @Override + public String getDataSourceName() { + return "STARROCKS"; + } @Override public FormStructure filterOptionRule( @@ -51,6 +59,8 @@ public FormStructure filterOptionRule( BusinessMode businessMode, PluginType pluginType, OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, List excludedKeys) { if (PluginType.SOURCE.equals(pluginType)) { throw new UnsupportedOperationException("Unsupported PluginType: " + pluginType); @@ -66,6 +76,8 @@ public FormStructure filterOptionRule( businessMode, pluginType, connectorOptionRule, + addRequiredOptions, + addOptionalOptions, excludedKeys); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/TidbDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/TidbDataSourceConfigSwitcher.java index 46db6690f..b70e53bfb 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/TidbDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/TidbDataSourceConfigSwitcher.java @@ -25,10 +25,13 @@ import org.apache.seatunnel.app.domain.request.job.DataSourceOption; import org.apache.seatunnel.app.domain.request.job.SelectTableFields; import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; import org.apache.seatunnel.common.constants.PluginType; +import com.google.auto.service.AutoService; + +@AutoService(DataSourceConfigSwitcher.class) public class TidbDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitcher { - public static final TidbDataSourceConfigSwitcher INSTANCE = new TidbDataSourceConfigSwitcher(); private static final String FACTORY = "factory"; @@ -41,7 +44,12 @@ public class TidbDataSourceConfigSwitcher extends BaseJdbcDataSourceConfigSwitch private static final String DATABASE_NAMES = "database-names"; - private TidbDataSourceConfigSwitcher() {} + public TidbDataSourceConfigSwitcher() {} + + @Override + public String getDataSourceName() { + return "JDBC-TIDB"; + } @Override public Config mergeDatasourceConfig( diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java index 294db5ed0..20d39e828 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/engine/SeaTunnelEngineProxy.java @@ -24,10 +24,12 @@ import com.hazelcast.client.config.ClientConfig; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; import java.util.Map; import java.util.concurrent.ExecutionException; +@Slf4j public class SeaTunnelEngineProxy { ClientConfig clientConfig = null; @@ -70,6 +72,9 @@ public String getJobStatus(@NonNull String jobEngineId) { SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig); try { return seaTunnelClient.getJobStatus(Long.valueOf(jobEngineId)); + } catch (Exception e) { + log.warn("Can not get job from engine.", e); + return null; } finally { seaTunnelClient.close(); } @@ -92,8 +97,7 @@ public String getAllRunningJobMetricsContent() { SeaTunnelClient seaTunnelClient = new SeaTunnelClient(clientConfig); try { - /*return seaTunnelClient.getJobClient().getRunningJobMetrics();*/ - return "waitting ST 2.3.3"; + return seaTunnelClient.getJobClient().getRunningJobMetrics(); } finally { seaTunnelClient.close(); } diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/framework/SeaTunnelOptionRuleWrapper.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/framework/SeaTunnelOptionRuleWrapper.java index 7a0bfb6ea..4f5e7c609 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/framework/SeaTunnelOptionRuleWrapper.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/framework/SeaTunnelOptionRuleWrapper.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.RequiredOption; import org.apache.seatunnel.app.dynamicforms.AbstractFormOption; +import org.apache.seatunnel.app.dynamicforms.Constants; import org.apache.seatunnel.app.dynamicforms.FormLocale; import org.apache.seatunnel.app.dynamicforms.FormOptionBuilder; import org.apache.seatunnel.app.dynamicforms.FormStructure; @@ -40,6 +41,8 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.seatunnel.app.common.SeaTunnelConnectorI18n.CONNECTOR_I18N_CONFIG_EN; @@ -74,20 +77,14 @@ public static FormStructure wrapper( @NonNull String name) { FormLocale locale = new FormLocale(); List optionFormOptions = wrapperOptionOptions(name, optionList, locale); - List> requiredFormOptions = + List requiredFormOptions = wrapperRequiredOptions(name, requiredList, locale); FormStructureBuilder formStructureBuilder = FormStructure.builder().name(name); if (!CollectionUtils.isEmpty(requiredFormOptions)) { - requiredFormOptions.forEach( - list -> { - if (CollectionUtils.isEmpty(list)) { - return; - } - - formStructureBuilder.addFormOption(list.toArray(new AbstractFormOption[1])); - }); + formStructureBuilder.addFormOption( + requiredFormOptions.toArray(new AbstractFormOption[1])); } if (!CollectionUtils.isEmpty(optionFormOptions)) { @@ -110,167 +107,177 @@ private static List wrapperOptionOptions( .collect(Collectors.toList()); } - private static List> wrapperRequiredOptions( + private static List wrapperRequiredOptions( @NonNull String connectorName, @NonNull List requiredList, FormLocale locale) { - List> formOptionsList = - requiredList.stream() - .map( + List result = new ArrayList<>(); + requiredList.forEach( + requiredOptions -> { + if (requiredOptions instanceof RequiredOption.AbsolutelyRequiredOptions) { + RequiredOption.AbsolutelyRequiredOptions absolutelyRequiredOptions = + (RequiredOption.AbsolutelyRequiredOptions) requiredOptions; + absolutelyRequiredOptions + .getRequiredOption() + .forEach( + option -> { + AbstractFormOption requiredFormItem = null; + for (AbstractFormOption formItem : result) { + if (formItem.getField().equals(option.key())) { + requiredFormItem = formItem; + break; + } + } + + if (requiredFormItem == null) { + requiredFormItem = + wrapperToFormOption( + connectorName, option, locale); + result.add(requiredFormItem); + } + + if (requiredFormItem.getValidate() == null) { + requiredFormItem.withValidate( + ValidateBuilder.builder() + .nonEmptyValidateBuilder() + .nonEmptyValidate()); + } + }); + } else if (requiredOptions instanceof RequiredOption.BundledRequiredOptions) { + List> bundledRequiredOptions = + ((RequiredOption.BundledRequiredOptions) requiredOptions) + .getRequiredOption(); + List bundledFields = + bundledRequiredOptions.stream() + .map(requiredOption -> requiredOption.key()) + .collect(Collectors.toList()); + + bundledRequiredOptions.forEach( option -> { - if (option - instanceof RequiredOption.AbsolutelyRequiredOptions) { - List collect = - ((RequiredOption.AbsolutelyRequiredOptions) option) - .getRequiredOption().stream() - .map( - requiredOption -> { - return wrapperToFormOption( - connectorName, - requiredOption, - locale) - .withValidate( - ValidateBuilder - .builder() - .nonEmptyValidateBuilder() - .nonEmptyValidate()); - }) - .collect(Collectors.toList()); - return collect; + AbstractFormOption bundledRequiredFormOption = null; + for (AbstractFormOption formItem : result) { + if (formItem.getField().equals(option.key())) { + bundledRequiredFormOption = formItem; + break; + } } - if (option instanceof RequiredOption.BundledRequiredOptions) { - List> bundledRequiredOptions = - ((RequiredOption.BundledRequiredOptions) option) - .getRequiredOption(); - List bundledFields = - bundledRequiredOptions.stream() - .map(requiredOption -> requiredOption.key()) - .collect(Collectors.toList()); - - List collect = - bundledRequiredOptions.stream() - .map( - requiredOption -> { - AbstractFormOption - bundledRequiredFormOption = - wrapperToFormOption( - connectorName, - requiredOption, - locale); - bundledRequiredFormOption - .withValidate( - ValidateBuilder - .builder() - .unionNonEmptyValidateBuilder() - .fields( - bundledFields - .toArray( - new String - [1])) - .unionNonEmptyValidate()); - return bundledRequiredFormOption; - }) - .collect(Collectors.toList()); - return collect; + if (bundledRequiredFormOption == null) { + bundledRequiredFormOption = + wrapperToFormOption(connectorName, option, locale); + result.add(bundledRequiredFormOption); } - if (option instanceof RequiredOption.ExclusiveRequiredOptions) { - List> exclusiveOptions = - ((RequiredOption.ExclusiveRequiredOptions) option) - .getExclusiveOptions(); - List exclusiveFields = - exclusiveOptions.stream() - .map(requiredOption -> requiredOption.key()) - .collect(Collectors.toList()); - - List collect = - exclusiveOptions.stream() - .map( - requiredOption -> { - AbstractFormOption - exclusiveRequiredFormOption = - wrapperToFormOption( - connectorName, - requiredOption, - locale) - .withValidate( - ValidateBuilder - .builder() - .mutuallyExclusiveValidateBuilder() - .fields( - exclusiveFields - .toArray( - new String - [1])) - .mutuallyExclusiveValidate()); - return exclusiveRequiredFormOption; - }) - .collect(Collectors.toList()); - return collect; + bundledRequiredFormOption.withValidate( + ValidateBuilder.builder() + .unionNonEmptyValidateBuilder() + .fields(bundledFields.toArray(new String[1])) + .unionNonEmptyValidate()); + }); + } else if (requiredOptions instanceof RequiredOption.ExclusiveRequiredOptions) { + List> exclusiveOptions = + ((RequiredOption.ExclusiveRequiredOptions) requiredOptions) + .getExclusiveOptions(); + List exclusiveFields = + exclusiveOptions.stream() + .map(requiredOption -> requiredOption.key()) + .collect(Collectors.toList()); + + exclusiveOptions.forEach( + option -> { + AbstractFormOption exclusiveRequiredFormOption = null; + for (AbstractFormOption formItem : result) { + if (formItem.getField().equals(option.key())) { + exclusiveRequiredFormOption = formItem; + break; + } } + if (exclusiveRequiredFormOption == null) { + exclusiveRequiredFormOption = + wrapperToFormOption(connectorName, option, locale); - if (option - instanceof RequiredOption.ConditionalRequiredOptions) { - RequiredOption.ConditionalRequiredOptions - conditionalRequiredOptions = - (RequiredOption.ConditionalRequiredOptions) - option; - - // we only support one field to control a form option, so we - // only need get condition key from the - // first expression. And all expression is 'or' and every - // condition have the same key. - String conditionKey = - conditionalRequiredOptions - .getExpression() - .getCondition() - .getOption() - .key(); - List expectValueList = new ArrayList<>(); - Expression expression = - conditionalRequiredOptions.getExpression(); - expectValueList.add( - expression - .getCondition() - .getExpectValue() - .toString()); - while (expression.hasNext()) { - expression = expression.getNext(); - expectValueList.add( - expression - .getCondition() - .getExpectValue() - .toString()); - } - List collect = - conditionalRequiredOptions.getRequiredOption() - .stream() - .map( - requiredOption -> { - return wrapperToFormOption( - connectorName, - requiredOption, - locale) - .withShow( - conditionKey, - expectValueList) - .withValidate( - ValidateBuilder - .builder() - .nonEmptyValidateBuilder() - .nonEmptyValidate()); - }) - .collect(Collectors.toList()); - return collect; + result.add(exclusiveRequiredFormOption); } - throw new UnSupportWrapperException( - connectorName, "Unknown", option.toString()); - }) - .collect(Collectors.toList()); + exclusiveRequiredFormOption.withValidate( + ValidateBuilder.builder() + .mutuallyExclusiveValidateBuilder() + .fields(exclusiveFields.toArray(new String[1])) + .mutuallyExclusiveValidate()); + }); + } else if (requiredOptions + instanceof RequiredOption.ConditionalRequiredOptions) { + RequiredOption.ConditionalRequiredOptions conditionalRequiredOptions = + (RequiredOption.ConditionalRequiredOptions) requiredOptions; + + // we only support one field to control a form option, so we + // only need get condition key from the + // first expression. And all expression is 'or' and every + // condition have the same key. + String conditionKey = + conditionalRequiredOptions + .getExpression() + .getCondition() + .getOption() + .key(); + List expectValueList = new ArrayList<>(); + Expression expression = conditionalRequiredOptions.getExpression(); + expectValueList.add(expression.getCondition().getExpectValue().toString()); + while (expression.hasNext()) { + expression = expression.getNext(); + expectValueList.add( + expression.getCondition().getExpectValue().toString()); + } - return formOptionsList; + conditionalRequiredOptions + .getRequiredOption() + .forEach( + option -> { + AbstractFormOption conditionalRequiredFormItem = null; + for (AbstractFormOption formItem : result) { + if (formItem.getField().equals(option.key())) { + conditionalRequiredFormItem = formItem; + break; + } + } + + if (conditionalRequiredFormItem == null) { + conditionalRequiredFormItem = + wrapperToFormOption( + connectorName, option, locale); + result.add(conditionalRequiredFormItem); + } + + if (conditionalRequiredFormItem.getShow() == null) { + conditionalRequiredFormItem.withShow( + conditionKey, expectValueList); + } else { + Map show = + conditionalRequiredFormItem.getShow(); + String field = + show.get(Constants.SHOW_FIELD).toString(); + if (field.equals(conditionKey)) { + Set values = + (Set) show.get(Constants.SHOW_VALUE); + values.addAll(expectValueList); + } else { + throw new UnSupportWrapperException( + connectorName, + conditionalRequiredFormItem.getLabel(), + "Only support show by one field"); + } + } + + if (conditionalRequiredFormItem.getValidate() == null) { + conditionalRequiredFormItem.withValidate( + ValidateBuilder.builder() + .nonEmptyValidateBuilder() + .nonEmptyValidate()); + } + }); + } + }); + return result; } private static AbstractFormOption wrapperToFormOption( diff --git a/seatunnel-server/seatunnel-app/src/main/resources/application.yml b/seatunnel-server/seatunnel-app/src/main/resources/application.yml index e35d8d132..a85295a6f 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/application.yml +++ b/seatunnel-server/seatunnel-app/src/main/resources/application.yml @@ -24,27 +24,18 @@ spring: jackson: date-format: yyyy-MM-dd HH:mm:ss datasource: - driver-class-name: com.mysql.jdbc.Driver + driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://localhost:3306/seatunnel?useSSL=false&useUnicode=true&characterEncoding=utf-8&allowMultiQueries=true&allowPublicKeyRetrieval=true username: root - password: root@123 + password: 123456 mvc: pathmatch: matching-strategy: ant_path_matcher -ds: - script: - dir: /dj - project: - default: seatunnel_test - tenant: - default: gaojun - api: - token: 6781b46bba1039c622d602beb25e880e - prefix: http://localhost:12345/dolphinscheduler jwt: expireTime: 86400 - secretKey: https://github.com/apache/incubator-seatunnel + # please add key when deploy + secretKey: algorithm: HS256 --- @@ -54,9 +45,16 @@ spring: on-profile: h2 sql: init: - schema-locations: classpath:script/seatunnel_server_h2.sql + schema-locations: classpath*:script/seatunnel_server_h2.sql datasource: driver-class-name: org.h2.Driver url: jdbc:h2:mem:seatunnel;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true username: sa - password: "" \ No newline at end of file + password: sa + h2: + console: + enabled: true + path: /h2 + settings: + trace: false + web-allow-others: false \ No newline at end of file diff --git a/seatunnel-server/seatunnel-app/src/main/resources/banner.txt b/seatunnel-server/seatunnel-app/src/main/resources/banner.txt new file mode 100644 index 000000000..0c7de9fe3 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/resources/banner.txt @@ -0,0 +1,8 @@ + + // ) ) /__ ___/ || / | / / + (( ___ ___ / / __ __ ___ // || / | / / ___ / __ + \\ //___) ) // ) ) / / // / / // ) ) // ) ) //___) ) // || / /||/ / //___) ) // ) ) + ) ) // // / / / / // / / // / / // / / // // ||/ / | / // // / / +((___ / / ((____ ((___( ( / / ((___( ( // / / // / / ((____ // | / | / ((____ ((___/ / +:: SeaTunnel web :: ${} + diff --git a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml index e72bf1e7d..4bcd6fd0f 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml +++ b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml @@ -20,13 +20,12 @@ connector-datasource-mapper: Jdbc: dataSources: - JDBC-Mysql - - JDBC-Redshift - JDBC-Postgres - JDBC-TiDB - - JDBC-StarRocks - JDBC-SQLServer - JDBC-KingBase - JDBC-Oracle + - JDBC-Db2 MySQL-CDC: dataSources: @@ -35,7 +34,6 @@ connector-datasource-mapper: Kafka: dataSources: - Kafka - - Kafka-Kingbase StarRocks: dataSources: @@ -49,58 +47,36 @@ connector-datasource-mapper: dataSources: - S3 - S3Redshift: - dataSources: - - S3-Redshift - SqlServer-CDC: dataSources: - SqlServer-CDC - Oracle-CDC: - dataSources: - - Oracle-CDC - Postgres-CDC: dataSources: - Postgres-CDC + MongoDB: + dataSources: + - MongoDB + sourceDatasourceFeatures: JDBC-Mysql: businessMode: - DATA_INTEGRATION sceneMode: - SINGLE_TABLE - JDBC-Redshift: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE JDBC-Postgres: businessMode: - DATA_INTEGRATION sceneMode: - SINGLE_TABLE - JDBC-TiDB: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE - JDBC-StarRocks: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE + JDBC-SQLServer: businessMode: - DATA_INTEGRATION sceneMode: - SINGLE_TABLE - JDBC-KingBase: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE + JDBC-Oracle: businessMode: - DATA_INTEGRATION @@ -142,11 +118,6 @@ connector-datasource-mapper: sceneMode: - SINGLE_TABLE - S3-Redshift: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE SqlServer-CDC: businessMode: @@ -157,40 +128,34 @@ connector-datasource-mapper: - MULTIPLE_TABLE - SPLIT_TABLE - Kafka-Kingbase: + MongoDB: businessMode: - DATA_INTEGRATION - - DATA_REPLICA sceneMode: - SINGLE_TABLE - - MULTIPLE_TABLE - - SPLIT_TABLE - - Oracle-CDC: + jobMode: + - BATCH + JDBC-Db2: businessMode: - DATA_INTEGRATION - - DATA_REPLICA sceneMode: - SINGLE_TABLE - - MULTIPLE_TABLE - - SPLIT_TABLE sinkDatasourceFeatures: JDBC-Mysql: businessMode: - DATA_INTEGRATION + - DATA_REPLICA sceneMode: - SINGLE_TABLE - JDBC-Redshift: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE + - MULTIPLE_TABLE JDBC-Postgres: businessMode: - DATA_INTEGRATION + - DATA_REPLICA sceneMode: - SINGLE_TABLE + - MULTIPLE_TABLE JDBC-TiDB: businessMode: - DATA_INTEGRATION @@ -199,26 +164,26 @@ connector-datasource-mapper: - SINGLE_TABLE - MULTIPLE_TABLE - SPLIT_TABLE - JDBC-StarRocks: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE JDBC-SQLServer: businessMode: - DATA_INTEGRATION + - DATA_REPLICA sceneMode: - SINGLE_TABLE + - MULTIPLE_TABLE JDBC-KingBase: businessMode: - DATA_INTEGRATION + - DATA_REPLICA sceneMode: - SINGLE_TABLE JDBC-Oracle: businessMode: - DATA_INTEGRATION + - DATA_REPLICA sceneMode: - SINGLE_TABLE + - MULTIPLE_TABLE Kafka: businessMode: - DATA_REPLICA @@ -248,7 +213,14 @@ connector-datasource-mapper: sceneMode: - SINGLE_TABLE - S3-Redshift: + MongoDB: + businessMode: + - DATA_INTEGRATION + - DATA_REPLICA + sceneMode: + - SINGLE_TABLE + - MULTIPLE_TABLE + JDBC-Db2: businessMode: - DATA_INTEGRATION sceneMode: diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml deleted file mode 100644 index 588637d24..000000000 --- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ProcessTaskRelationMapper.xml +++ /dev/null @@ -1,213 +0,0 @@ - - - - - - - - id, name, process_definition_version, project_code, process_definition_code, pre_task_code, pre_task_version, - post_task_code, post_task_version, condition_type, condition_params, create_time, update_time - - - - - - delete from t_ds_process_task_relation - WHERE project_code = #{projectCode} - and process_definition_code = #{processCode} - - - insert into t_ds_process_task_relation (name, process_definition_version, project_code, process_definition_code, - pre_task_code, pre_task_version, post_task_code, post_task_version, condition_type, condition_params, create_time, update_time) - values - - (#{relation.name},#{relation.processDefinitionVersion},#{relation.projectCode},#{relation.processDefinitionCode}, - #{relation.preTaskCode},#{relation.preTaskVersion},#{relation.postTaskCode},#{relation.postTaskVersion}, - #{relation.conditionType},#{relation.conditionParams},#{relation.createTime},#{relation.updateTime}) - - - - - - - - - - - - - - - - - - update t_ds_process_task_relation - - pre_task_code=#{processTaskRelation.preTaskCode}, - pre_task_version=#{processTaskRelation.preTaskVersion} - - - WHERE id = #{processTaskRelation.id} - - - - - - - - delete from t_ds_process_task_relation - WHERE project_code = #{processTaskRelationLog.projectCode} - and process_definition_code = #{processTaskRelationLog.processDefinitionCode} - and process_definition_version = #{processTaskRelationLog.processDefinitionVersion} - - and pre_task_code = #{processTaskRelationLog.preTaskCode} - and pre_task_version = #{processTaskRelationLog.preTaskVersion} - - and post_task_code = #{processTaskRelationLog.postTaskCode} - and post_task_version = #{processTaskRelationLog.postTaskVersion} - - - - - - diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/SchedulerConfigMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/SchedulerConfigMapper.xml deleted file mode 100644 index 30a6ad933..000000000 --- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/SchedulerConfigMapper.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - - - - - - - - - - - id, - script_id, - trigger_expression, - retry_times, - retry_interval, - active_start_time, - active_end_time, - create_time, - update_time, - creator_id, - update_id - - - - delete - from scheduler_config - where id = #{id,jdbcType=INTEGER} - - - insert into scheduler_config (script_id, trigger_expression, retry_times, retry_interval, - active_start_time, active_end_time, create_time, - update_time, creator_id, update_id) - values (#{scriptId,jdbcType=INTEGER}, #{retryTimes,jdbcType=INTEGER}, #{retryInterval,jdbcType=INTEGER}, - #{activeStartTime,jdbcType=TIMESTAMP}, #{activeEndTime,jdbcType=TIMESTAMP}, - #{createTime,jdbcType=TIMESTAMP}, - #{updateTime,jdbcType=TIMESTAMP}, #{creatorId,jdbcType=INTEGER}, #{updateId,jdbcType=INTEGER}) - - - diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptJobApplyMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptJobApplyMapper.xml deleted file mode 100644 index 310901563..000000000 --- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptJobApplyMapper.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - id, - script_id, - scheduler_config_id, - job_id, - operator_id, - create_time, - update_time - - - a.script_id, - a.scheduler_config_id, - a.job_id, - a.operator_id, - a.create_time, - a.update_time, - b.trigger_expression, - b.retry_times, - b.retry_interval, - b.active_start_time, - b.active_end_time - - - - delete - from script_job_apply - where id = #{id,jdbcType=INTEGER} - - - insert into script_job_apply (script_id, scheduler_config_id, - job_id, operator_id) - values (#{scriptId,jdbcType=INTEGER}, #{schedulerConfigId,jdbcType=INTEGER}, - #{jobId,jdbcType=BIGINT}, #{operatorId,jdbcType=INTEGER}) - - - update script_job_apply - set script_id = #{scriptId,jdbcType=INTEGER}, - scheduler_config_id = #{schedulerConfigId,jdbcType=INTEGER}, - job_id = #{jobId}, - operator_id = #{operatorId,jdbcType=INTEGER} - - - - - - diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptMapper.xml deleted file mode 100644 index a481b3a8f..000000000 --- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptMapper.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - - - id, - `name`, - `type`, - `status`, - content, - content_md5, - creator_id, - mender_id, - create_time, - update_time - - - insert into `script` (name, type, status, creator_id, mender_id, content) - VALUES (#{name,jdbcType=VARCHAR}, #{type,jdbcType=TINYINT}, #{status,jdbcType=TINYINT}, - #{creatorId,jdbcType=INTEGER}, - #{menderId,jdbcType=INTEGER}, - #{content,jdbcType=VARCHAR}) - - - update `script` - set `content` = #{content,jdbcType=VARCHAR}, - `content_md5` = #{contentMd5,jdbcType=VARCHAR}, - `mender_id` = #{menderId, jdbcType=INTEGER} - where id = #{id,jdbcType=INTEGER} - - - update `script` - set `status` = #{code,jdbcType=TINYINT} - where id = #{id,jdbcType=INTEGER} - - - - - - diff --git a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptParamMapper.xml b/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptParamMapper.xml deleted file mode 100644 index 6971fc8cd..000000000 --- a/seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptParamMapper.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - id, - script_id, - `key`, - `value`, - `status`, - create_time, - update_time - - - insert into `script_param` (`script_id`, `key`, `value`, `status`) - values - - (#{item.scriptId,jdbcType=INTEGER}, #{item.key,jdbcType=VARCHAR}, #{item.value,jdbcType=VARCHAR}, - #{item.status,jdbcType=TINYINT}) - - - - update `script_param` - set `status` = #{code, jdbcType=TINYINT} - where `script_id` = #{scriptId, jdbcType=INTEGER} - - - - diff --git a/seatunnel-server/seatunnel-app/src/main/resources/plugin-mapping.properties b/seatunnel-server/seatunnel-app/src/main/resources/plugin-mapping.properties new file mode 100644 index 000000000..551da2c7c --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/resources/plugin-mapping.properties @@ -0,0 +1,115 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This mapping is used to resolve the Jar package name without version (or call artifactId) +# corresponding to the module in the user Config, helping SeaTunnel to load the correct Jar package. + +## *** WARNING **** : `seatunnel.source.XXX`, the `XXX` should be string which SeaTunnelSource::getPluginName and TableSinkFactory::factoryIdentifier returned value## + +# SeaTunnel Connector-V2 + +seatunnel.source.FakeSource = connector-fake +seatunnel.sink.Console = connector-console +seatunnel.sink.Assert = connector-assert +seatunnel.source.Kafka = connector-kafka +seatunnel.sink.Kafka = connector-kafka +seatunnel.source.Http = connector-http-base +seatunnel.sink.Http = connector-http-base +seatunnel.sink.Feishu = connector-http-feishu +seatunnel.source.Socket = connector-socket +seatunnel.sink.Hive = connector-hive +seatunnel.source.Hive = connector-hive +seatunnel.source.Clickhouse = connector-clickhouse +seatunnel.sink.Clickhouse = connector-clickhouse +seatunnel.sink.ClickhouseFile = connector-clickhouse +seatunnel.source.Jdbc = connector-jdbc +seatunnel.sink.Jdbc = connector-jdbc +seatunnel.source.Kudu = connector-kudu +seatunnel.sink.Kudu = connector-kudu +seatunnel.sink.EmailSink = connector-email +seatunnel.source.HdfsFile = connector-file-hadoop +seatunnel.sink.HdfsFile = connector-file-hadoop +seatunnel.source.LocalFile = connector-file-local +seatunnel.sink.LocalFile = connector-file-local +seatunnel.source.OssFile = connector-file-oss +seatunnel.sink.OssFile = connector-file-oss +seatunnel.source.OssJindoFile = connector-file-jindo-oss +seatunnel.sink.OssJindoFile = connector-file-jindo-oss +seatunnel.source.CosFile = connector-file-cos +seatunnel.sink.CosFile = connector-file-cos +seatunnel.source.Pulsar = connector-pulsar +seatunnel.source.Hudi = connector-hudi +seatunnel.sink.DingTalk = connector-dingtalk +seatunnel.source.Elasticsearch = connector-elasticsearch +seatunnel.sink.Elasticsearch = connector-elasticsearch +seatunnel.source.IoTDB = connector-iotdb +seatunnel.sink.IoTDB = connector-iotdb +seatunnel.source.Neo4j = connector-neo4j +seatunnel.sink.Neo4j = connector-neo4j +seatunnel.source.FtpFile = connector-file-ftp +seatunnel.sink.FtpFile = connector-file-ftp +seatunnel.source.SftpFile = connector-file-sftp +seatunnel.sink.SftpFile = connector-file-sftp +seatunnel.sink.Socket = connector-socket +seatunnel.source.Redis = connector-redis +seatunnel.sink.Redis = connector-redis +seatunnel.sink.DataHub = connector-datahub +seatunnel.sink.Sentry = connector-sentry +seatunnel.source.MongoDB = connector-mongodb +seatunnel.sink.MongoDB = connector-mongodb +seatunnel.source.Iceberg = connector-iceberg +seatunnel.source.InfluxDB = connector-influxdb +seatunnel.source.S3File = connector-file-s3 +seatunnel.sink.S3File = connector-file-s3 +seatunnel.source.AmazonDynamodb = connector-amazondynamodb +seatunnel.sink.AmazonDynamodb = connector-amazondynamodb +seatunnel.source.Cassandra = connector-cassandra +seatunnel.sink.Cassandra = connector-cassandra +seatunnel.sink.StarRocks = connector-starrocks +seatunnel.source.MyHours = connector-http-myhours +seatunnel.sink.InfluxDB = connector-influxdb +seatunnel.source.GoogleSheets = connector-google-sheets +seatunnel.sink.GoogleFirestore = connector-google-firestore +seatunnel.sink.Tablestore = connector-tablestore +seatunnel.source.Lemlist = connector-http-lemlist +seatunnel.source.Klaviyo = connector-http-klaviyo +seatunnel.sink.Slack = connector-slack +seatunnel.source.OneSignal = connector-http-onesignal +seatunnel.source.Jira = connector-http-jira +seatunnel.source.Gitlab = connector-http-gitlab +seatunnel.source.Github = connector-http-github +seatunnel.source.Notion = connector-http-notion +seatunnel.sink.RabbitMQ = connector-rabbitmq +seatunnel.source.RabbitMQ = connector-rabbitmq +seatunnel.source.OpenMldb = connector-openmldb +seatunnel.source.SqlServer-CDC = connector-cdc-sqlserver +seatunnel.sink.Doris = connector-doris +seatunnel.source.Maxcompute = connector-maxcompute +seatunnel.sink.Maxcompute = connector-maxcompute +seatunnel.source.MySQL-CDC = connector-cdc-mysql +seatunnel.source.MongoDB-CDC = connector-cdc-mongodb +seatunnel.sink.S3Redshift = connector-s3-redshift +seatunnel.source.TDengine = connector-tdengine +seatunnel.sink.TDengine = connector-tdengine +seatunnel.source.Persistiq = connector-http-persistiq +seatunnel.sink.SelectDBCloud = connector-selectdb-cloud +seatunnel.sink.Hbase = connector-hbase +seatunnel.source.StarRocks = connector-starrocks +seatunnel.source.Rocketmq = connector-rocketmq +seatunnel.sink.Rocketmq = connector-rocketmq +seatunnel.source.Paimon = connector-paimon +seatunnel.sink.Paimon = connector-paimon diff --git a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql index a38ffe3c2..d74a0bdca 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql +++ b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_h2.sql @@ -15,161 +15,210 @@ * limitations under the License. */ --- CREATE DATABASE IF NOT EXISTS seatunnel; - +DROP TABLE IF EXISTS "user_login_log"; +CREATE TABLE "user_login_log" ( + id BIGINT NOT NULL AUTO_INCREMENT, + user_id INT NOT NULL, + token CLOB NOT NULL, + token_status TINYINT NOT NULL, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); -- ---------------------------- -- Table structure for role -- ---------------------------- -DROP TABLE IF EXISTS `role` CASCADE;; -CREATE TABLE `role` ( - `id` int(20) NOT NULL AUTO_INCREMENT, - `type` int(2) NOT NULL, - `role_name` varchar(255) NOT NULL, - `description` varchar(255) DEFAULT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +DROP TABLE IF EXISTS role; +CREATE TABLE role ( + id INT AUTO_INCREMENT PRIMARY KEY, + type INT NOT NULL, + role_name VARCHAR(255) NOT NULL, + description VARCHAR(255), + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ); --- ---------------------------- --- Records of role --- ---------------------------- - -- ---------------------------- -- Table structure for role_user_relation -- ---------------------------- -DROP TABLE IF EXISTS `role_user_relation` CASCADE; -CREATE TABLE `role_user_relation` ( - `id` int(20) NOT NULL AUTO_INCREMENT, - `role_id` int(20) NOT NULL, - `user_id` int(20) NOT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +DROP TABLE IF EXISTS role_user_relation; +CREATE TABLE role_user_relation ( + id INT AUTO_INCREMENT PRIMARY KEY, + role_id INT NOT NULL, + user_id INT NOT NULL, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ); --- ---------------------------- --- Records of role_user_relation --- ---------------------------- +-- Table structure for t_st_datasource +DROP TABLE IF EXISTS t_st_datasource; +CREATE TABLE t_st_datasource ( + id BIGINT NOT NULL, + datasource_name VARCHAR(63) NOT NULL, + plugin_name VARCHAR(63) NOT NULL, + plugin_version VARCHAR(63) DEFAULT '1.0.0', + datasource_config VARCHAR(1023) NOT NULL, + description VARCHAR(63) DEFAULT NULL, + create_user_id INT NOT NULL, + update_user_id INT NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + UNIQUE (datasource_name) +); --- ---------------------------- --- Table structure for scheduler_config --- ---------------------------- -DROP TABLE IF EXISTS `scheduler_config` CASCADE; -CREATE TABLE `scheduler_config` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) DEFAULT NULL, - `trigger_expression` varchar(255) DEFAULT NULL, - `retry_times` int(11) NOT NULL DEFAULT '0', - `retry_interval` int(11) NOT NULL DEFAULT '0', - `active_start_time` datetime(3) NOT NULL, - `active_end_time` datetime(3) NOT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - `creator_id` int(11) NOT NULL, - `update_id` int(11) NOT NULL, - PRIMARY KEY (`id`) +-- Table structure for t_st_job_definition +DROP TABLE IF EXISTS t_st_job_definition; +CREATE TABLE t_st_job_definition ( + id BIGINT NOT NULL, + name VARCHAR(50) NOT NULL, + description VARCHAR(255) DEFAULT NULL, + job_type VARCHAR(50) DEFAULT NULL, + create_user_id INT NOT NULL, + update_user_id INT NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + UNIQUE (name) ); --- ---------------------------- --- Records of scheduler_config --- ---------------------------- --- ---------------------------- --- Table structure for script --- ---------------------------- -DROP TABLE IF EXISTS `script` CASCADE; -CREATE TABLE `script` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL, - `type` tinyint(4) NOT NULL, - `status` tinyint(4) NOT NULL, - `content` mediumtext, - `content_md5` varchar(255) DEFAULT NULL, - `creator_id` int(11) NOT NULL, - `mender_id` int(11) NOT NULL, - `create_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +-- Table structure for t_st_job_instance +DROP TABLE IF EXISTS t_st_job_instance; +CREATE TABLE t_st_job_instance ( + id BIGINT NOT NULL, + job_define_id BIGINT NOT NULL, + job_status VARCHAR(50) DEFAULT NULL, + job_config CLOB NOT NULL, + engine_name VARCHAR(50) NOT NULL, + engine_version VARCHAR(50) NOT NULL, + job_engine_id VARCHAR(200) DEFAULT NULL, + create_user_id INT NOT NULL, + update_user_id INT DEFAULT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + end_time TIMESTAMP(3) DEFAULT NULL, + job_type VARCHAR(50) NOT NULL, + PRIMARY KEY (id) ); --- ---------------------------- --- Records of script --- ---------------------------- +-- Table structure for t_st_job_instance_history +DROP TABLE IF EXISTS t_st_job_instance_history; +CREATE TABLE t_st_job_instance_history ( + id BIGINT NOT NULL, + dag CLOB NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id) +); --- ---------------------------- --- Table structure for script_job_apply --- ---------------------------- -DROP TABLE IF EXISTS `script_job_apply` CASCADE; -CREATE TABLE `script_job_apply` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) NOT NULL, - `scheduler_config_id` int(11) NOT NULL, - `job_id` bigint(20) DEFAULT NULL, - `operator_id` int(11) NOT NULL, - `create_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +-- Table structure for t_st_job_line +DROP TABLE IF EXISTS t_st_job_line; +CREATE TABLE t_st_job_line ( + id BIGINT NOT NULL, + version_id BIGINT NOT NULL, + input_plugin_id VARCHAR(50) NOT NULL, + target_plugin_id VARCHAR(50) NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + INDEX job_line_version_index (version_id) ); --- ---------------------------- --- Records of script_job_apply --- ---------------------------- +-- Table structure for t_st_job_metrics +DROP TABLE IF EXISTS t_st_job_metrics; +CREATE TABLE t_st_job_metrics ( + id BIGINT NOT NULL, + job_instance_id BIGINT NOT NULL, + pipeline_id INT NOT NULL, + read_row_count BIGINT NOT NULL, + write_row_count BIGINT NOT NULL, + source_table_names VARCHAR(200) DEFAULT NULL, + sink_table_names VARCHAR(200) DEFAULT NULL, + read_qps BIGINT DEFAULT NULL, + write_qps BIGINT DEFAULT NULL, + record_delay BIGINT DEFAULT NULL, + status VARCHAR(20) DEFAULT NULL, + create_user_id INT NOT NULL, + update_user_id INT DEFAULT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id) +); --- ---------------------------- --- Table structure for script_param --- ---------------------------- -DROP TABLE IF EXISTS `script_param` CASCADE; -CREATE TABLE `script_param` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) DEFAULT NULL, - `key` varchar(255) NOT NULL, - `value` varchar(255) DEFAULT NULL, - `status` tinyint(4) DEFAULT NULL, - `create_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +-- Table structure for t_st_job_task +DROP TABLE IF EXISTS t_st_job_task; +CREATE TABLE t_st_job_task ( + id BIGINT NOT NULL, + version_id BIGINT NOT NULL, + plugin_id VARCHAR(50) NOT NULL, + name VARCHAR(50) NOT NULL, + config CLOB DEFAULT NULL, + transform_options VARCHAR(5000) DEFAULT NULL, + output_schema CLOB DEFAULT NULL, + connector_type VARCHAR(50) NOT NULL, + datasource_id BIGINT DEFAULT NULL, + datasource_option VARCHAR(5000) DEFAULT NULL, + select_table_fields VARCHAR(5000) DEFAULT NULL, + scene_mode VARCHAR(50) DEFAULT NULL, + type VARCHAR(50) NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + INDEX job_task_plugin_id_index (plugin_id) ); --- ---------------------------- --- Records of script_param --- ---------------------------- +-- Table structure for t_st_job_version +DROP TABLE IF EXISTS t_st_job_version; +CREATE TABLE t_st_job_version ( + id BIGINT NOT NULL, + job_id BIGINT NOT NULL, + name VARCHAR(255) NOT NULL, + job_mode VARCHAR(10) NOT NULL, + env CLOB DEFAULT NULL, + engine_name VARCHAR(50) NOT NULL, + engine_version VARCHAR(50) NOT NULL, + create_user_id INT NOT NULL, + update_user_id INT NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id) +); + +-- Table structure for t_st_virtual_table +DROP TABLE IF EXISTS t_st_virtual_table; +CREATE TABLE t_st_virtual_table ( + id BIGINT NOT NULL, + datasource_id BIGINT NOT NULL, + virtual_database_name VARCHAR(63) NOT NULL, + virtual_table_name VARCHAR(63) NOT NULL, + table_fields VARCHAR(1023) NOT NULL, + virtual_table_config VARCHAR(1023) NOT NULL, + description VARCHAR(63) DEFAULT NULL, + create_user_id INT NOT NULL, + update_user_id INT NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id) +); --- ---------------------------- -- Table structure for user --- ---------------------------- -DROP TABLE IF EXISTS `user` CASCADE; -CREATE TABLE `user` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `username` varchar(255) NOT NULL, - `password` varchar(255) NOT NULL, - `status` tinyint(4) NOT NULL, - `type` tinyint(4) NOT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) +DROP TABLE IF EXISTS "user"; +CREATE TABLE "user" ( + id INT NOT NULL AUTO_INCREMENT, + username VARCHAR(255) NOT NULL, + password VARCHAR(255) NOT NULL, + status TINYINT NOT NULL, + type TINYINT NOT NULL, + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + PRIMARY KEY (id) ); --- ---------------------------- -- Records of user --- ---------------------------- - --- ---------------------------- --- Table structure for user_login_log --- ---------------------------- -DROP TABLE IF EXISTS `user_login_log` CASCADE; -CREATE TABLE `user_login_log` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `user_id` int(11) NOT NULL, - `token` mediumtext NOT NULL, - `token_status` tinyint(1) NOT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) -); +INSERT INTO "user" ("username", "password", "status", "type") VALUES ('admin', '7f97da8846fed829bb8d1fd9f8030f3b', 0, 0); --- ---------------------------- -- Records of user_login_log --- ---------------------------- - -INSERT INTO `seatunnel`.`user`(`username`,`password`,`status`,`type`) values ('admin', '7f97da8846fed829bb8d1fd9f8030f3b', 0, 0); +-- No equivalent records provided for the user_login_log table in the provided SQL script. +-- You can insert records into this table using similar INSERT INTO statements. +-- However, you would need to provide the values for columns like "user_id", "token", "token_status", etc. diff --git a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql index f4abea506..4f6663d64 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql +++ b/seatunnel-server/seatunnel-app/src/main/resources/script/seatunnel_server_mysql.sql @@ -15,6 +15,7 @@ * limitations under the License. */ +CREATE DATABASE IF NOT EXISTS seatunnel; use seatunnel; SET NAMES utf8mb4; SET FOREIGN_KEY_CHECKS = 0; @@ -33,6 +34,12 @@ CREATE TABLE `role` ( PRIMARY KEY (`id`) USING BTREE ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; +-- ---------------------------- +-- Records of role +-- ---------------------------- +INSERT INTO `seatunnel`.`role`(`type`,`role_name`,`description`) values (0, 'ADMIN_ROLE', 'Admin User'); +INSERT INTO `seatunnel`.`role`(`type`,`role_name`,`description`) values (1, 'NORMAL_ROLE', 'Normal User'); + -- ---------------------------- -- Table structure for role_user_relation -- ---------------------------- @@ -46,212 +53,6 @@ CREATE TABLE `role_user_relation` ( PRIMARY KEY (`id`) USING BTREE ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; --- ---------------------------- --- Table structure for scheduler_config --- ---------------------------- -DROP TABLE IF EXISTS `scheduler_config`; -CREATE TABLE `scheduler_config` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) NULL DEFAULT NULL, - `trigger_expression` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, - `retry_times` int(11) NOT NULL DEFAULT 0, - `retry_interval` int(11) NOT NULL DEFAULT 0, - `active_start_time` datetime(3) NOT NULL, - `active_end_time` datetime(3) NOT NULL, - `create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - `creator_id` int(11) NOT NULL, - `update_id` int(11) NOT NULL, - PRIMARY KEY (`id`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for script --- ---------------------------- -DROP TABLE IF EXISTS `script`; -CREATE TABLE `script` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, - `type` tinyint(4) NOT NULL, - `status` tinyint(4) NOT NULL, - `content` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL, - `content_md5` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, - `creator_id` int(11) NOT NULL, - `mender_id` int(11) NOT NULL, - `create_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 4 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for script_job_apply --- ---------------------------- -DROP TABLE IF EXISTS `script_job_apply`; -CREATE TABLE `script_job_apply` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) NOT NULL, - `scheduler_config_id` int(11) NOT NULL, - `job_id` bigint(20) NULL DEFAULT NULL, - `operator_id` int(11) NOT NULL, - `create_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for script_param --- ---------------------------- -DROP TABLE IF EXISTS `script_param`; -CREATE TABLE `script_param` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `script_id` int(11) NULL DEFAULT NULL, - `key` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, - `value` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, - `status` tinyint(4) NULL DEFAULT NULL, - `create_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3), - `update_time` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (`id`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for t_ds_process_definition --- ---------------------------- -DROP TABLE IF EXISTS `t_ds_process_definition`; -CREATE TABLE `t_ds_process_definition` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id', - `code` bigint(20) NOT NULL COMMENT 'encoding', - `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'process definition name', - `version` int(11) NULL DEFAULT 0 COMMENT 'process definition version', - `description` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'description', - `release_state` tinyint(4) NULL DEFAULT NULL COMMENT 'process definition release state：0:offline,1:online', - `user_id` int(11) NULL DEFAULT NULL COMMENT 'process definition creator id', - `global_params` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'global parameters', - `flag` tinyint(4) NULL DEFAULT NULL COMMENT '0 not available, 1 available', - `locations` longtext CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'Node location information', - `warning_group_id` int(11) NULL DEFAULT NULL COMMENT 'alert group id', - `timeout` int(11) NULL DEFAULT 0 COMMENT 'time out, unit: minute', - `tenant_id` int(11) NOT NULL DEFAULT -1 COMMENT 'tenant id', - `execution_type` tinyint(4) NULL DEFAULT 0 COMMENT 'execution_type 0:parallel,1:serial wait,2:serial discard,3:serial priority', - `create_time` datetime NOT NULL COMMENT 'create time', - `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `idx_code`(`code`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for t_ds_process_task_relation --- ---------------------------- -DROP TABLE IF EXISTS `t_ds_process_task_relation`; -CREATE TABLE `t_ds_process_task_relation` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id', - `name` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'relation name', - `project_code` bigint(20) NOT NULL COMMENT 'project code', - `process_definition_code` bigint(20) NOT NULL COMMENT 'process code', - `process_definition_version` int(11) NOT NULL COMMENT 'process version', - `pre_task_code` bigint(20) NOT NULL COMMENT 'pre task code', - `pre_task_version` int(11) NOT NULL COMMENT 'pre task version', - `post_task_code` bigint(20) NOT NULL COMMENT 'post task code', - `post_task_version` int(11) NOT NULL COMMENT 'post task version', - `condition_type` tinyint(2) NULL DEFAULT NULL COMMENT 'condition type : 0 none, 1 judge 2 delay', - `condition_params` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'condition params(json)', - `create_time` datetime NOT NULL COMMENT 'create time', - `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) USING BTREE, - INDEX `idx_code`(`project_code`, `process_definition_code`) USING BTREE, - INDEX `idx_pre_task_code_version`(`pre_task_code`, `pre_task_version`) USING BTREE, - INDEX `idx_post_task_code_version`(`post_task_code`, `post_task_version`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for t_ds_process_task_relation_log --- ---------------------------- -DROP TABLE IF EXISTS `t_ds_process_task_relation_log`; -CREATE TABLE `t_ds_process_task_relation_log` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id', - `name` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'relation name', - `project_code` bigint(20) NOT NULL COMMENT 'project code', - `process_definition_code` bigint(20) NOT NULL COMMENT 'process code', - `process_definition_version` int(11) NOT NULL COMMENT 'process version', - `pre_task_code` bigint(20) NOT NULL COMMENT 'pre task code', - `pre_task_version` int(11) NOT NULL COMMENT 'pre task version', - `post_task_code` bigint(20) NOT NULL COMMENT 'post task code', - `post_task_version` int(11) NOT NULL COMMENT 'post task version', - `condition_type` tinyint(2) NULL DEFAULT NULL COMMENT 'condition type : 0 none, 1 judge 2 delay', - `condition_params` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'condition params(json)', - `operator` int(11) NULL DEFAULT NULL COMMENT 'operator user id', - `operate_time` datetime NULL DEFAULT NULL COMMENT 'operate time', - `create_time` datetime NOT NULL COMMENT 'create time', - `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) USING BTREE, - INDEX `idx_process_code_version`(`process_definition_code`, `process_definition_version`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for t_ds_task_definition --- ---------------------------- -DROP TABLE IF EXISTS `t_ds_task_definition`; -CREATE TABLE `t_ds_task_definition` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id', - `code` bigint(20) NOT NULL COMMENT 'encoding', - `name` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT 'task definition name', - `version` int(11) NULL DEFAULT 0 COMMENT 'task definition version', - `description` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'description', - `user_id` int(11) NULL DEFAULT NULL COMMENT 'task definition creator id', - `task_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'task type', - `task_params` longtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'job custom parameters', - `flag` tinyint(2) NULL DEFAULT NULL COMMENT '0 not available, 1 available', - `task_priority` tinyint(4) NULL DEFAULT 2 COMMENT 'job priority', - `worker_group` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT 'worker grouping', - `environment_code` bigint(20) NULL DEFAULT -1 COMMENT 'environment code', - `fail_retry_times` int(11) NULL DEFAULT NULL COMMENT 'number of failed retries', - `fail_retry_interval` int(11) NULL DEFAULT NULL COMMENT 'failed retry interval', - `timeout_flag` tinyint(2) NULL DEFAULT 0 COMMENT 'timeout flag:0 close, 1 open', - `timeout_notify_strategy` tinyint(4) NULL DEFAULT NULL COMMENT 'timeout notification policy: 0 warning, 1 fail', - `timeout` int(11) NULL DEFAULT 0 COMMENT 'timeout length,unit: minute', - `delay_time` int(11) NULL DEFAULT 0 COMMENT 'delay execution time,unit: minute', - `resource_ids` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT 'resource id, separated by comma', - `task_group_id` int(11) NULL DEFAULT NULL COMMENT 'task group id', - `task_group_priority` tinyint(4) NULL DEFAULT 0 COMMENT 'task group priority', - `create_time` datetime NOT NULL COMMENT 'create time', - `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_code`(`code`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; - --- ---------------------------- --- Table structure for t_ds_task_definition_log --- ---------------------------- -DROP TABLE IF EXISTS `t_ds_task_definition_log`; -CREATE TABLE `t_ds_task_definition_log` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id', - `code` bigint(20) NOT NULL COMMENT 'encoding', - `name` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'task definition name', - `version` int(11) NULL DEFAULT 0 COMMENT 'task definition version', - `description` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'description', - `user_id` int(11) NULL DEFAULT NULL COMMENT 'task definition creator id', - `task_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'task type', - `task_params` longtext CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'job custom parameters', - `flag` tinyint(2) NULL DEFAULT NULL COMMENT '0 not available, 1 available', - `task_priority` tinyint(4) NULL DEFAULT 2 COMMENT 'job priority', - `worker_group` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'worker grouping', - `environment_code` bigint(20) NULL DEFAULT -1 COMMENT 'environment code', - `fail_retry_times` int(11) NULL DEFAULT NULL COMMENT 'number of failed retries', - `fail_retry_interval` int(11) NULL DEFAULT NULL COMMENT 'failed retry interval', - `timeout_flag` tinyint(2) NULL DEFAULT 0 COMMENT 'timeout flag:0 close, 1 open', - `timeout_notify_strategy` tinyint(4) NULL DEFAULT NULL COMMENT 'timeout notification policy: 0 warning, 1 fail', - `timeout` int(11) NULL DEFAULT 0 COMMENT 'timeout length,unit: minute', - `delay_time` int(11) NULL DEFAULT 0 COMMENT 'delay execution time,unit: minute', - `resource_ids` text CHARACTER SET utf8 COLLATE utf8_bin NULL COMMENT 'resource id, separated by comma', - `operator` int(11) NULL DEFAULT NULL COMMENT 'operator user id', - `task_group_id` int(11) NULL DEFAULT NULL COMMENT 'task group id', - `task_group_priority` tinyint(4) NULL DEFAULT 0 COMMENT 'task group priority', - `operate_time` datetime NULL DEFAULT NULL COMMENT 'operate time', - `create_time` datetime NOT NULL COMMENT 'create time', - `update_time` datetime NOT NULL COMMENT 'update time', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_code_version`(`code`, `version`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Dynamic; - -- ---------------------------- -- Table structure for t_st_datasource -- ---------------------------- diff --git a/seatunnel-server/seatunnel-dynamicform/pom.xml b/seatunnel-server/seatunnel-dynamicform/pom.xml index 4655bb0b8..1f414a7dc 100644 --- a/seatunnel-server/seatunnel-dynamicform/pom.xml +++ b/seatunnel-server/seatunnel-dynamicform/pom.xml @@ -34,16 +34,6 @@ org.apache.commons commons-lang3 - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.jupiter - junit-jupiter-params - test - org.apache.seatunnel seatunnel-common diff --git a/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/AbstractFormOption.java b/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/AbstractFormOption.java index bc3e65c20..2f829b580 100644 --- a/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/AbstractFormOption.java +++ b/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/AbstractFormOption.java @@ -27,8 +27,10 @@ import lombok.NonNull; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; @Data public abstract class AbstractFormOption { @@ -75,8 +77,9 @@ public T withShow(@NonNull String field, @NonNull List values) { this.show = new HashMap<>(); } + Set valueSet = new HashSet<>(values); this.show.put(Constants.SHOW_FIELD, field); - this.show.put(Constants.SHOW_VALUE, values); + this.show.put(Constants.SHOW_VALUE, valueSet); return (T) this; } diff --git a/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/FormStructureValidate.java b/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/FormStructureValidate.java index 74080b645..532c3691a 100644 --- a/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/FormStructureValidate.java +++ b/seatunnel-server/seatunnel-dynamicform/src/main/java/org/apache/seatunnel/app/dynamicforms/FormStructureValidate.java @@ -42,11 +42,13 @@ public static void validateFormStructure(@NonNull FormStructure formStructure) List showErrorList = validateShow(formStructure); List unionNonErrorList = validateUnionNonEmpty(formStructure); List exclusiveErrorList = validateMutuallyExclusive(formStructure); + List duplicateFormItemErrorList = validateDuplicateFormItem(formStructure); apiErrorList.addAll(localeErrorList); apiErrorList.addAll(showErrorList); apiErrorList.addAll(unionNonErrorList); apiErrorList.addAll(exclusiveErrorList); + apiErrorList.addAll(duplicateFormItemErrorList); if (apiErrorList.size() > 0) { throw new FormStructureValidateException(formStructure.getName(), apiErrorList); @@ -267,4 +269,28 @@ private static List validateMutuallyExclusive(@NonNull FormStructure for return errorMessageList; } + + public static List validateDuplicateFormItem(@NonNull FormStructure formStructure) { + List fieldList = new ArrayList(); + List errorFieldList = new ArrayList(); + List errorMessage = new ArrayList(); + formStructure + .getForms() + .forEach( + form -> { + if (fieldList.contains(form.getField())) { + errorFieldList.add(form.getField()); + } else { + fieldList.add(form.getField()); + } + }); + if (errorFieldList.size() > 0) { + errorMessage.add( + String.format( + "DuplicateFormItemValidate failed, Duplicate form items %s", + errorFieldList)); + } + + return errorMessage; + } } diff --git a/seatunnel-server/seatunnel-scheduler/pom.xml b/seatunnel-server/seatunnel-scheduler/pom.xml deleted file mode 100644 index 229371b25..000000000 --- a/seatunnel-server/seatunnel-scheduler/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-server - ${revision} - - - seatunnel-scheduler - pom - - seatunnel-scheduler-dolphinscheduler - - - - - org.apache.seatunnel - seatunnel-spi - ${project.version} - - - - diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/pom.xml b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/pom.xml deleted file mode 100644 index a31c92a16..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-scheduler - ${revision} - - - seatunnel-scheduler-dolphinscheduler - - - 1.8 - - - - - org.springframework - spring-aop - ${spring.version} - provided - - - org.springframework - spring-context - ${spring.version} - provided - - - - org.jsoup - jsoup - - - com.google.guava - guava - provided - - - org.apache.commons - commons-collections4 - - - com.fasterxml.jackson.core - jackson-databind - provided - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - - - diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/ExecuteTypeEnum.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/ExecuteTypeEnum.java deleted file mode 100644 index f8b77a877..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/ExecuteTypeEnum.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler; - -public enum ExecuteTypeEnum { - NONE, - REPEAT_RUNNING, - RECOVER_SUSPENDED_PROCESS, - START_FAILURE_TASK_PROCESS, - STOP, - PAUSE -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/IDolphinschedulerService.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/IDolphinschedulerService.java deleted file mode 100644 index 713c5d8ff..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/IDolphinschedulerService.java +++ /dev/null @@ -1,72 +0,0 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package org.apache.seatunnel.scheduler.dolphinscheduler; -// -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListTaskInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ResourceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.SchedulerDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.StartProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.UpdateProcessDefinitionDto; -// import org.apache.seatunnel.server.common.PageData; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceLogDto; -// import org.apache.seatunnel.spi.scheduler.dto.JobDto; -// -// import java.util.List; -// -// public interface IDolphinschedulerService { -// -// ProcessDefinitionDto createOrUpdateProcessDefinition(UpdateProcessDefinitionDto dto); -// -// PageData listProcessDefinition(ListProcessDefinitionDto dto); -// -// ProcessDefinitionDto fetchProcessDefinitionByName(String processDefinitionName); -// -// void startProcessDefinition(StartProcessDefinitionDto dto); -// -// void updateProcessDefinitionState( -// long processDefinitionCode, String processDefinitionName, String state); -// -// SchedulerDto createOrUpdateSchedule(JobDto dto); -// -// List listSchedule(long processDefinitionCode); -// -// void scheduleOnline(int scheduleId); -// -// void scheduleOffline(int scheduleId); -// -// List genTaskCodes(long projectCode, int num); -// -// ResourceDto createOrUpdateScriptContent(String resourceName, String content); -// -// PageData listTaskInstance(ListTaskInstanceDto dto); -// -// void deleteProcessDefinition(long code); -// -// PageData listProcessInstance(ListProcessInstanceDto dto); -// -// InstanceLogDto queryInstanceLog(long instanceId); -// -// void killProcessInstance(long processInstanceId); -// -// void execute(long processInstanceId, ExecuteTypeEnum executeType); -// } diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/constants/DolphinschedulerConstants.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/constants/DolphinschedulerConstants.java deleted file mode 100644 index 3220d894a..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/constants/DolphinschedulerConstants.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.constants; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class DolphinschedulerConstants { - - /** api url */ - public static final String QUERY_PROJECT_LIST_PAGING = "/projects"; - - public static final String QUERY_LIST_PAGING = "/projects/%s/process-definition"; - public static final String GEN_TASK_CODE_LIST = "/projects/%s/task-definition/gen-task-codes"; - public static final String VERIFY_RESOURCE_NAME = "/resources/verify-name"; - public static final String QUERY_RESOURCE = "/resources/%s"; - public static final String ONLINE_CREATE_RESOURCE = "/resources/online-create"; - public static final String UPDATE_CONTENT = "/resources/%s/update-content"; - public static final String LOG_DETAIL = "/log/detail"; - public static final String RELEASE = "/projects/%s/process-definition/%s/release"; - public static final String START_PROCESS_INSTANCE = - "/projects/%s/executors/start-process-instance"; - public static final String QUERY_PROCESS_DEFINITION_BY_NAME = - "/projects/%s/process-definition/query-by-name"; - public static final String QUERY_TASK_LIST_PAGING = "/projects/%s/task-instances"; - public static final String CREATE_PROCESS_DEFINITION = "/projects/%s/process-definition/"; - public static final String CREATE_SCHEDULE = "/projects/%s/schedules/"; - public static final String QUERY_SCHEDULE_LIST_PAGING = "/projects/%s/schedules"; - public static final String SCHEDULE_ONLINE = "/projects/%s/schedules/%s/online"; - public static final String SCHEDULE_OFFLINE = "/projects/%s/schedules/%s/offline"; - public static final String DELETE_PROCESS_DEFINITION = "/projects/%s/process-definition/%s"; - public static final String PROCESS_INSTANCE_LIST = "/projects/%s/process-instances"; - public static final String EXECUTE = "/projects/%s/executors/execute"; - - /** request param */ - public static final String TOKEN = "token"; - - public static final String SEARCH_VAL = "searchVal"; - public static final String PAGE_SIZE = "pageSize"; - public static final int PAGE_SIZE_DEFAULT = 10; - public static final int PAGE_SIZE_MIN = 1; - public static final String PAGE_NO = "pageNo"; - public static final int PAGE_NO_DEFAULT = 1; - public static final String GEN_NUM = "genNum"; - public static final int GEN_NUM_DEFAULT = 1; - public static final String RESOURCE_SEPARATOR = "/"; - public static final String FULL_NAME = "fullName"; - public static final String RESOURCE_TYPE = "type"; - public static final String RESOURCE_TYPE_FILE = "FILE"; - public static final String RESOURCE_TYPE_FILE_SUFFIX = "suffix"; - public static final String RESOURCE_TYPE_FILE_SUFFIX_DEFAULT = "conf"; - public static final String RESOURCE_TYPE_FILE_CONTENT = "content"; - public static final String RESOURCE_TYPE_UDF = "UDF"; - public static final int RESOURCE_ID_DEFAULT = 0; - public static final String RESOURCE_ID = "id"; - public static final String TASK_INSTANCE_ID = "taskInstanceId"; - public static final String LOG_SKIP_LINE_NUM = "skipLineNum"; - public static final String LOG_LIMIT_NUM = "limit"; - public static final String PROCESS_DEFINITION_NAME = "name"; - public static final String RELEASE_STATE = "releaseState"; - public static final String RELEASE_STATE_ONLINE = "ONLINE"; - public static final String RELEASE_STATE_OFFLINE = "OFFLINE"; - public static final String FAILURE_STRATEGY = "failureStrategy"; - public static final String FAILURE_STRATEGY_DEFAULT = "CONTINUE"; - public static final String WORKER_GROUP = "workerGroup"; - public static final String WORKER_GROUP_DEFAULT = "default"; - public static final String WARNING_TYPE = "warningType"; - public static final String WARNING_TYPE_DEFAULT = "NONE"; - public static final String WARNING_GROUP_ID = "warningGroupId"; - public static final int WARNING_GROUP_ID_DEFAULT = 0; - public static final String TASK_DEPEND_TYPE_DEFAULT = "TASK_POST"; - public static final String RUN_MODE_DEFAULT = "RUN_MODE_SERIAL"; - public static final String RUN_MODE_PARALLEL = "RUN_MODE_PARALLEL"; - public static final String PROCESS_INSTANCE_PRIORITY = "processInstancePriority"; - public static final String PROCESS_INSTANCE_PRIORITY_DEFAULT = "MEDIUM"; - public static final int DRY_RUN = 0; - public static final String PROCESS_DEFINITION = "processDefinition"; - public static final String PROCESS_INSTANCE_NAME = "processInstanceName"; - public static final String LOCATIONS = "locations"; - public static final String LOCATIONS_X = "x"; - public static final int LOCATIONS_X_DEFAULT = 200; - public static final String LOCATIONS_Y = "y"; - public static final int LOCATIONS_Y_DEFAULT = 200; - public static final String TASK_CODE = "taskCode"; - public static final String TASK_DEFINITION_JSON = "taskDefinitionJson"; - public static final String TASK_RELATION_JSON = "taskRelationJson"; - public static final String TENANT_CODE = "tenantCode"; - public static final String TASK_RELATION_JSON_NAME = "name"; - public static final String PRE_TASK_CODE = "preTaskCode"; - public static final long PRE_TASK_CODE_DEFAULT = 0; - public static final String PRE_TASK_VERSION = "preTaskVersion"; - public static final int PRE_TASK_VERSION_DEFAULT = 0; - public static final String POST_TASK_CODE = "postTaskCode"; - public static final String POST_TASK_VERSION = "postTaskVersion"; - public static final int POST_TASK_VERSION_DEFAULT = 1; - public static final String CONDITION_TYPE = "conditionType"; - public static final int CONDITION_TYPE_DEFAULT = 0; - public static final String CONDITION_PARAMS = "conditionParams"; - public static final String TASK_DEFINITION_JSON_CODE = "code"; - public static final String TASK_DEFINITION_JSON_NAME = "name"; - public static final String VERSION = "version"; - public static final int VERSION_DEFAULT = 1; - public static final String DESCRIPTION = "description"; - public static final String DESCRIPTION_DEFAULT = ""; - public static final String DELAY_TIME = "delayTime"; - public static final int DELAY_TIME_DEFAULT = 0; - public static final String TASK_TYPE = "taskType"; - public static final String TASK_TYPE_DEFAULT = "SHELL"; - public static final String TASK_PARAMS = "taskParams"; - public static final String RESOURCE_LIST = "resourceList"; - public static final String LOCAL_PARAMS = "localParams"; - public static final String LOCAL_PARAMS_PROP = "prop"; - public static final String LOCAL_PARAMS_DIRECT = "direct"; - public static final String LOCAL_PARAMS_DIRECT_DEFAULT = "IN"; - public static final String LOCAL_PARAMS_TYPE = "type"; - public static final String LOCAL_PARAMS_TYPE_DEFAULT = "VARCHAR"; - public static final String LOCAL_PARAMS_VALUE = "value"; - public static final String RAW_SCRIPT = "rawScript"; - public static final String DEPENDENCE = "dependence"; - public static final Map DEPENDENCE_DEFAULT = Collections.emptyMap(); - public static final String CONDITION_RESULT = "conditionResult"; - public static final String SUCCESS_NODE = "successNode"; - public static final List SUCCESS_NODE_DEFAULT = Collections.emptyList(); - public static final String FAILED_NODE = "failedNode"; - public static final List FAILED_NODE_DEFAULT = Collections.emptyList(); - public static final String WAIT_START_TIMEOUT = "waitStartTimeout"; - public static final int WAIT_START_TIMEOUT_DEFAULT = 0; - public static final String SWITCH_RESULT = "switchResult"; - public static final int SWITCH_RESULT_DEFAULT = 0; - public static final String FLAG = "flag"; - public static final String FLAG_DEFAULT = "YES"; - public static final String TASK_PRIORITY = "taskPriority"; - public static final String TASK_PRIORITY_DEFAULT = "MEDIUM"; - public static final String FAIL_RETRY_TIMES = "failRetryTimes"; - public static final String FAIL_RETRY_INTERVAL = "failRetryInterval"; - public static final String TIMEOUT_FLAG = "timeoutFlag"; - public static final String TIMEOUT_FLAG_DEFAULT = "CLOSE"; - public static final String TIMEOUT_NOTIFY_STRATEGY = "timeoutNotifyStrategy"; - public static final String TIMEOUT_NOTIFY_STRATEGY_DEFAULT = "WARN"; - public static final String TIMEOUT = "timeout"; - public static final int TIMEOUT_DEFAULT = 0; - public static final String ENVIRONMENT_CODE = "environmentCode"; - public static final int ENVIRONMENT_CODE_DEFAULT = -1; - public static final String PROCESS_DEFINITION_CODE = "processDefinitionCode"; - public static final String START_TIME = "startTime"; - public static final String END_TIME = "endTime"; - public static final String CRONTAB = "crontab"; - public static final String TIMEZONE_ID = "timezoneId"; - public static final String TIMEZONE_ID_DEFAULT = "Asia/Shanghai"; - public static final String SCHEDULE = "schedule"; - public static final String SCHEDULE_ID = "id"; - public static final String DEFAULT_FILE_SUFFIX = ".conf"; - public static final String EXEC_TYPE_DEFAULT = "START_PROCESS"; - public static final String EXEC_TYPE_COMPLEMENT = "COMPLEMENT_DATA"; - public static final String DEPENDENT_MODE_DEFAULT = "OFF_MODE"; - public static final String PROCESS_INSTANCE_ID = "processInstanceId"; - public static final String EXECUTE_TYPE = "executeType"; - public static final int LOG_SKIP_LINE_NUM_DEFAULT = 0; - public static final int LOG_LIMIT_NUM_DEFAULT = Integer.MAX_VALUE; - public static final String STATE_TYPE = "stateType"; - - /** response param */ - public static final String DATA = "data"; - - public static final String DATA_TOTAL_LIST = "totalList"; - public static final String DATA_TOTAL = "total"; - public static final String CODE = "code"; - public static final int CODE_SUCCESS = 0; - public static final String MSG = "msg"; - public static final String LOG_MESSAGE = "message"; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/BaseListDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/BaseListDto.java deleted file mode 100644 index 844e9ab83..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/BaseListDto.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Data; - -@Data -public class BaseListDto { - protected String name; - protected int pageNo; - protected int pageSize; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ConditionResult.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ConditionResult.java deleted file mode 100644 index 629b2be9d..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ConditionResult.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -import java.util.List; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class ConditionResult { - private List successNode; - private List failedNode; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessDefinitionDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessDefinitionDto.java deleted file mode 100644 index acae1c807..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessDefinitionDto.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Data; - -@Data -public class ListProcessDefinitionDto extends BaseListDto {} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessInstanceDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessInstanceDto.java deleted file mode 100644 index 5c6f64478..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListProcessInstanceDto.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Data; - -@Data -public class ListProcessInstanceDto extends BaseListDto {} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListTaskInstanceDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListTaskInstanceDto.java deleted file mode 100644 index 531e10171..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ListTaskInstanceDto.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Data; - -@Data -public class ListTaskInstanceDto extends BaseListDto {} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocalParam.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocalParam.java deleted file mode 100644 index 4596d02fb..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocalParam.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class LocalParam { - private String prop; - private String direct; - private String type; - private Object value; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocationDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocationDto.java deleted file mode 100644 index 33fa65389..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/LocationDto.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class LocationDto { - private Long taskCode; - private int x; - private int y; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessDefinitionDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessDefinitionDto.java deleted file mode 100644 index 6a2f76e7a..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessDefinitionDto.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -import java.util.Date; - -import static org.apache.seatunnel.server.common.DateUtils.DEFAULT_DATETIME_FORMAT; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class ProcessDefinitionDto { - private int id; - private long code; - private String name; - private String releaseState; - private long projectCode; - private String description; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date createTime; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date updateTime; - - private String userName; - private String projectName; - private String locations; - private String scheduleReleaseState; - private int timeout; - private int tenantId; - private String tenantCode; - private String modifyBy; - private int warningGroupId; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessInstanceDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessInstanceDto.java deleted file mode 100644 index 2e4311696..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProcessInstanceDto.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.Data; - -import java.util.Date; - -import static org.apache.seatunnel.server.common.DateUtils.DEFAULT_DATETIME_FORMAT; - -@Data -public class ProcessInstanceDto { - private int id; - private long processDefinitionCode; - private int processDefinitionVersion; - private String state; - private String recovery; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date startTime; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date endTime; - - private int runTimes; - private String name; - private String host; - private String processDefinition; - private String commandType; - private String commandParam; - private String taskDependType; - private int maxTryTimes; - private String failureStrategy; - private String warningType; - private String warningGroupId; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date scheduleTime; - - private String commandStartTime; - private String globalParams; - private String dagData; - private int executorId; - private String executorName; - private String tenantCode; - private String queue; - private String isSubProcess; - private String locations; - private String historyCmd; - private String dependenceScheduleTimes; - private String duration; - private String processInstancePriority; - private String workerGroup; - private String environmentCode; - private int timeout; - private int tenantId; - private String varPool; - private int nextProcessInstanceId; - private int dryRun; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date restartTime; - - private String cmdTypeIfComplement; - private boolean complementData; - private boolean blocked; - private boolean processInstanceStop; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProjectDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProjectDto.java deleted file mode 100644 index edcf1998b..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ProjectDto.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class ProjectDto { - private long code; - private String name; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ResourceDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ResourceDto.java deleted file mode 100644 index 8da5313c4..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/ResourceDto.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties( - ignoreUnknown = true, - value = {"pid"}) -public class ResourceDto { - private int id; - private int pid; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/SchedulerDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/SchedulerDto.java deleted file mode 100644 index 153826713..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/SchedulerDto.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.Data; - -import java.util.Date; - -import static org.apache.seatunnel.server.common.DateUtils.DEFAULT_DATETIME_FORMAT; - -@Data -public class SchedulerDto { - private int id; - private long processDefinitionCode; - private String processDefinitionName; - private String projectName; - private String definitionDescription; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date startTime; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date endTime; - - private String timezoneId; - private String crontab; - private String failureStrategy; - private String warningType; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date createTime; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date updateTime; - - private int userId; - private String userName; - private String releaseState; - private int warningGroupId; - private String processInstancePriority; - private String workerGroup; - private int environmentCode; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/StartProcessDefinitionDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/StartProcessDefinitionDto.java deleted file mode 100644 index 81e6d7579..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/StartProcessDefinitionDto.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.google.common.collect.Maps; -import lombok.Builder; -import lombok.Data; - -import java.util.Map; - -@Builder -@Data -public class StartProcessDefinitionDto { - private long processDefinitionCode; - private String failureStrategy; - private String warningType; - private int warningGroupId; - private String taskDependType; - private String runMode; - private String processInstancePriority; - private String workerGroup; - private int dryRun; - private String scheduleTime; - private String execType; - private String dependentMode; - private Integer expectedParallelismNumber; - - public Map toMap() { - final Map map = Maps.newHashMap(); - map.put("processDefinitionCode", String.valueOf(processDefinitionCode)); - map.put("failureStrategy", failureStrategy); - map.put("warningType", warningType); - map.put("warningGroupId", String.valueOf(warningGroupId)); - map.put("taskDependType", taskDependType); - map.put("runMode", runMode); - map.put("processInstancePriority", processInstancePriority); - map.put("workerGroup", workerGroup); - map.put("dryRun", String.valueOf(dryRun)); - map.put("scheduleTime", scheduleTime); - map.put("execType", execType); - map.put("dependentMode", dependentMode); - map.put("expectedParallelismNumber", String.valueOf(expectedParallelismNumber)); - return map; - } -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDefinitionDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDefinitionDto.java deleted file mode 100644 index c0b02569e..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskDefinitionDto.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskDefinitionDto { - private Long code; - private String name; - private String description; - private String taskType; - private TaskParamDto taskParams; - private String flag; - private String taskPriority; - private String workerGroup; - private int failRetryTimes; - private int failRetryInterval; - private String timeoutFlag; - private String timeoutNotifyStrategy; - private int timeout; - private int delayTime; - private int environmentCode; - private int version; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskInstanceDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskInstanceDto.java deleted file mode 100644 index b2026503a..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskInstanceDto.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -import java.util.Date; - -import static org.apache.seatunnel.server.common.DateUtils.DEFAULT_DATETIME_FORMAT; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskInstanceDto { - private boolean taskComplete; - private boolean firstRun; - private int dryRun; - private String flag; - private long environmentCode; - private String processInstance; - private int pid; - private String taskParams; - private String duration; - private String taskType; - private long taskCode; - private String taskInstancePriority; - private String host; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date startTime; - - private int id; - private String state; - private String workerGroup; - private String processInstancePriority; - private int processInstanceId; - private int executorId; - private String firstSubmitTime; - private String resources; - private int maxRetryTimes; - private int retryTimes; - private String executorName; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date submitTime; - - private String name; - private int retryInterval; - - @JsonFormat(pattern = DEFAULT_DATETIME_FORMAT) - private Date endTime; - - private String processInstanceName; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskParamDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskParamDto.java deleted file mode 100644 index 36ab04ee3..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskParamDto.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -import java.util.List; -import java.util.Map; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskParamDto { - private List resourceList; - private List localParams; - private String rawScript; - private Map dependence; - private ConditionResult conditionResult; - private int waitStartTimeout; - private int switchResult; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskRelationDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskRelationDto.java deleted file mode 100644 index c0c80b228..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/TaskRelationDto.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Data; - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class TaskRelationDto { - private String name; - private long preTaskCode; - private int preTaskVersion; - private long postTaskCode; - private long postTaskVersion; - private int conditionType; - private String conditionParams; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/UpdateProcessDefinitionDto.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/UpdateProcessDefinitionDto.java deleted file mode 100644 index 1486e4464..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/dto/UpdateProcessDefinitionDto.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.Date; - -@Data -@Builder -public class UpdateProcessDefinitionDto { - private Long processDefinitionCode; - private String name; - private Date startTime; - private Date endTime; - private String cronExpression; - private TaskDescriptionDto taskDescriptionDto; -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/RunFrequencyEnum.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/RunFrequencyEnum.java deleted file mode 100644 index f0bf58222..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/enums/RunFrequencyEnum.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.enums; - -public enum RunFrequencyEnum { - SCHEDULER, - ONCE, - ; - - public static final String SCHEDULER_TYPE = "SCHEDULER"; - - public static RunFrequencyEnum parse(String name) { - if (SCHEDULER_TYPE.equalsIgnoreCase(name)) { - return SCHEDULER; - } - return ONCE; - } -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/DolphinschedulerServiceImpl.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/DolphinschedulerServiceImpl.java deleted file mode 100644 index 1551b2df1..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/DolphinschedulerServiceImpl.java +++ /dev/null @@ -1,946 +0,0 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package org.apache.seatunnel.scheduler.dolphinscheduler.impl; -// -// import org.apache.seatunnel.scheduler.dolphinscheduler.ExecuteTypeEnum; -// import org.apache.seatunnel.scheduler.dolphinscheduler.IDolphinschedulerService; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ConditionResult; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListTaskInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.LocalParam; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.LocationDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.OnlineCreateResourceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProjectDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ResourceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.SchedulerDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.StartProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskDescriptionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskParamDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskRelationDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.UpdateProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.utils.HttpUtils; -// import org.apache.seatunnel.server.common.DateUtils; -// import org.apache.seatunnel.server.common.PageData; -// import org.apache.seatunnel.server.common.SeatunnelErrorEnum; -// import org.apache.seatunnel.server.common.SeatunnelException; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceLogDto; -// import org.apache.seatunnel.spi.scheduler.dto.JobDto; -// -// import org.apache.commons.collections4.MapUtils; -// -// import org.jsoup.Connection; -// import org.springframework.beans.factory.InitializingBean; -// import org.springframework.beans.factory.annotation.Value; -// import org.springframework.stereotype.Component; -// import org.springframework.util.CollectionUtils; -// -// import com.fasterxml.jackson.core.JsonProcessingException; -// import com.fasterxml.jackson.databind.DeserializationFeature; -// import com.fasterxml.jackson.databind.ObjectMapper; -// import com.google.common.collect.Lists; -// import com.google.common.collect.Maps; -// import lombok.extern.slf4j.Slf4j; -// -// import java.io.IOException; -// import java.util.Collections; -// import java.util.HashMap; -// import java.util.List; -// import java.util.Map; -// import java.util.Objects; -// import java.util.stream.Collectors; -// -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CODE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CODE_SUCCESS; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CONDITION_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CREATE_PROCESS_DEFINITION; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CREATE_SCHEDULE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.CRONTAB; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DATA; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DATA_TOTAL; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DATA_TOTAL_LIST; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DEFAULT_FILE_SUFFIX; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DELAY_TIME_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DELETE_PROCESS_DEFINITION; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DEPENDENCE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DESCRIPTION_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.END_TIME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.ENVIRONMENT_CODE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.ENVIRONMENT_CODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.EXECUTE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.EXECUTE_TYPE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FAILED_NODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FAILURE_STRATEGY; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FAILURE_STRATEGY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FLAG_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FULL_NAME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.GEN_NUM; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.GEN_NUM_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.GEN_TASK_CODE_LIST; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOCAL_PARAMS_DIRECT_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOCAL_PARAMS_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOCATIONS; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOCATIONS_X_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOCATIONS_Y_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_DETAIL; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_LIMIT_NUM; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_LIMIT_NUM_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_MESSAGE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_SKIP_LINE_NUM; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.LOG_SKIP_LINE_NUM_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.MSG; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.ONLINE_CREATE_RESOURCE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_NO; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_NO_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_SIZE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_SIZE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.POST_TASK_VERSION_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PRE_TASK_CODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PRE_TASK_VERSION_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_DEFINITION; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_DEFINITION_CODE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_DEFINITION_NAME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_LIST; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_NAME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_PRIORITY; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_PRIORITY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_LIST_PAGING; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_PROCESS_DEFINITION_BY_NAME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_PROJECT_LIST_PAGING; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_RESOURCE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_SCHEDULE_LIST_PAGING; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.QUERY_TASK_LIST_PAGING; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RELEASE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RELEASE_STATE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RELEASE_STATE_OFFLINE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_ID_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_SEPARATOR; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_TYPE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_TYPE_FILE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_TYPE_FILE_CONTENT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RESOURCE_TYPE_FILE_SUFFIX_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SCHEDULE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SCHEDULE_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SCHEDULE_OFFLINE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SCHEDULE_ONLINE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SEARCH_VAL; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.START_PROCESS_INSTANCE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.START_TIME; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SUCCESS_NODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.SWITCH_RESULT_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_DEFINITION_JSON; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_INSTANCE_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_PRIORITY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_RELATION_JSON; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TENANT_CODE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TIMEOUT_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TIMEOUT_FLAG_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TIMEOUT_NOTIFY_STRATEGY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TIMEZONE_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TIMEZONE_ID_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TOKEN; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.UPDATE_CONTENT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.VERSION_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WAIT_START_TIMEOUT_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_GROUP_ID; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_GROUP_ID_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_TYPE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WORKER_GROUP; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WORKER_GROUP_DEFAULT; -// import static org.apache.seatunnel.scheduler.dolphinscheduler.utils.HttpUtils.createParamMap; -// import static org.apache.seatunnel.server.common.Constants.BLANK_SPACE; -// import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.NO_MATCHED_PROJECT; -// import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.UNEXPECTED_RETURN_CODE; -// -// @Slf4j -// @Component -// public class DolphinschedulerServiceImpl implements IDolphinschedulerService, InitializingBean { -// -// @Value("${ds.api.prefix}") -// private String apiPrefix; -// -// @Value("${ds.api.token}") -// private String token; -// -// @Value("${ds.tenant.default}") -// private String defaultTenantName; -// -// @Value("${ds.project.default}") -// private String defaultProjectName; -// -// @Value("${ds.script.dir}") -// private String defaultScriptDir; -// -// private long defaultProjectCode; -// -// private static final ObjectMapper MAPPER = new ObjectMapper(); -// -// static { -// MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); -// } -// -// @Override -// public void afterPropertiesSet() throws Exception { -// final ProjectDto projectDto = queryProjectCodeByName(defaultProjectName); -// defaultProjectCode = projectDto.getCode(); -// } -// -// @Override -// public ProcessDefinitionDto createOrUpdateProcessDefinition(UpdateProcessDefinitionDto dto) { -// // gen task code -// final List taskCodes = genTaskCodes(defaultProjectCode, GEN_NUM_DEFAULT); -// -// // build taskDefinitionJson and taskRelationJson. -// final Long taskCode = taskCodes.get(0); -// List taskDefinitionJson = -// Collections.singletonList( -// buildTaskDefinitionJson(taskCode, dto.getTaskDescriptionDto())); -// List taskRelationJson = -// buildTaskRelationJson(taskCode, dto.getTaskDescriptionDto()); -// List locations = buildLocation(taskCodes); -// -// String url = apiPrefix.concat(String.format(CREATE_PROCESS_DEFINITION, -// defaultProjectCode)); -// Connection.Method method = Connection.Method.POST; -// if (Objects.nonNull(dto.getProcessDefinitionCode())) { -// method = Connection.Method.PUT; -// url = url.concat(String.valueOf(dto.getProcessDefinitionCode())); -// // offline process -// updateProcessDefinitionState( -// dto.getProcessDefinitionCode(), dto.getName(), RELEASE_STATE_OFFLINE); -// } -// -// final Map paramMap = -// createParamMap( -// LOCATIONS, -// locations, -// TASK_DEFINITION_JSON, -// this.objectToString(taskDefinitionJson), -// TASK_RELATION_JSON, -// this.objectToString(taskRelationJson), -// TENANT_CODE, -// defaultTenantName, -// PROCESS_DEFINITION_NAME, -// dto.getName()); -// -// final Map result = -// HttpUtils.builder() -// .withUrl(url) -// .withMethod(method) -// .withData(paramMap) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// -// checkResult(result, false); -// final Map map = MapUtils.getMap(result, DATA); -// return mapToPojo(map, ProcessDefinitionDto.class); -// } -// -// @Override -// public PageData listProcessDefinition(ListProcessDefinitionDto dto) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format(QUERY_LIST_PAGING, defaultProjectCode))) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// SEARCH_VAL, -// dto.getName(), -// PAGE_NO, -// dto.getPageNo(), -// PAGE_SIZE, -// dto.getPageSize())) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// final Map map = MapUtils.getMap(result, DATA); -// final int total = MapUtils.getIntValue(map, DATA_TOTAL); -// final List> processDefinitionList = -// (List>) map.get(DATA_TOTAL_LIST); -// -// if (CollectionUtils.isEmpty(processDefinitionList)) { -// return new PageData<>(total, Collections.emptyList()); -// } -// final List data = -// processDefinitionList.stream() -// .map(m -> this.mapToPojo(m, ProcessDefinitionDto.class)) -// .collect(Collectors.toList()); -// -// return new PageData<>(total, data); -// } -// -// @Override -// public ProcessDefinitionDto fetchProcessDefinitionByName(String processDefinitionName) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// QUERY_PROCESS_DEFINITION_BY_NAME, -// defaultProjectCode))) -// .withMethod(Connection.Method.GET) -// .withData(createParamMap(PROCESS_DEFINITION_NAME, processDefinitionName)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// final Map map = -// (Map) MapUtils.getMap(result, DATA).get(PROCESS_DEFINITION); -// return this.mapToPojo(map, ProcessDefinitionDto.class); -// } -// -// @Override -// public void startProcessDefinition(StartProcessDefinitionDto dto) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format(START_PROCESS_INSTANCE, -// defaultProjectCode))) -// .withMethod(Connection.Method.POST) -// .withData(dto.toMap()) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// @Override -// public void updateProcessDefinitionState( -// long processDefinitionCode, String processDefinitionName, String state) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// RELEASE, -// defaultProjectCode, -// processDefinitionCode))) -// .withMethod(Connection.Method.POST) -// .withData( -// createParamMap( -// PROCESS_DEFINITION_NAME, -// processDefinitionName, -// RELEASE_STATE, -// state)) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// @Override -// public SchedulerDto createOrUpdateSchedule(JobDto dto) { -// final Map map = Maps.newHashMap(); -// map.put(FAILURE_STRATEGY, FAILURE_STRATEGY_DEFAULT); -// map.put(WARNING_TYPE, WARNING_TYPE_DEFAULT); -// map.put(PROCESS_INSTANCE_PRIORITY, PROCESS_INSTANCE_PRIORITY_DEFAULT); -// map.put(WARNING_GROUP_ID, WARNING_GROUP_ID_DEFAULT); -// map.put(WORKER_GROUP, WORKER_GROUP_DEFAULT); -// map.put(ENVIRONMENT_CODE, ENVIRONMENT_CODE_DEFAULT); -// map.put(PROCESS_DEFINITION_CODE, dto.getJobId()); -// -// final Map schedule = Maps.newHashMap(); -// schedule.put(START_TIME, DateUtils.format(dto.getSchedulerConfigDto().getStartTime())); -// schedule.put(END_TIME, DateUtils.format(dto.getSchedulerConfigDto().getEndTime())); -// schedule.put(CRONTAB, dto.getSchedulerConfigDto().getTriggerExpression()); -// schedule.put(TIMEZONE_ID, TIMEZONE_ID_DEFAULT); -// -// map.put(SCHEDULE, this.objectToString(schedule)); -// -// String url = String.format(CREATE_SCHEDULE, defaultProjectCode); -// -// final List schedulerDtos = listSchedule(dto.getJobId()); -// boolean flag = false; -// if (!CollectionUtils.isEmpty(schedulerDtos)) { -// url = url.concat(String.valueOf(schedulerDtos.get(0).getId())); -// flag = true; -// } -// -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(url)) -// .withData(translate(map)) -// .withMethod(flag ? Connection.Method.PUT : Connection.Method.POST) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// if (flag) { -// schedulerDtos.clear(); -// schedulerDtos.addAll(listSchedule(dto.getJobId())); -// return schedulerDtos.get(0); -// } -// -// Map resultMap = MapUtils.getMap(result, DATA); -// return this.mapToPojo(resultMap, SchedulerDto.class); -// } -// -// @Override -// public List listSchedule(long processDefinitionCode) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// QUERY_SCHEDULE_LIST_PAGING, defaultProjectCode))) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// PROCESS_DEFINITION_CODE, -// processDefinitionCode, -// PAGE_NO, -// PAGE_NO_DEFAULT, -// PAGE_SIZE, -// PAGE_SIZE_DEFAULT)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// final List> scheduleList = -// (List>) MapUtils.getMap(result, DATA).get(DATA_TOTAL_LIST); -// if (CollectionUtils.isEmpty(scheduleList)) { -// return Collections.emptyList(); -// } -// return scheduleList.stream() -// .map(m -> this.mapToPojo(m, SchedulerDto.class)) -// .collect(Collectors.toList()); -// } -// -// @Override -// public void scheduleOnline(int scheduleId) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// SCHEDULE_ONLINE, defaultProjectCode, scheduleId))) -// .withMethod(Connection.Method.POST) -// .withData(createParamMap(SCHEDULE_ID, scheduleId)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// @Override -// public void scheduleOffline(int scheduleId) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// SCHEDULE_OFFLINE, defaultProjectCode, -// scheduleId))) -// .withMethod(Connection.Method.POST) -// .withData(createParamMap(SCHEDULE_ID, scheduleId)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// @Override -// public List genTaskCodes(long projectCode, int num) { -// final String url = String.format(GEN_TASK_CODE_LIST, projectCode); -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(url)) -// .withMethod(Connection.Method.GET) -// .withData(createParamMap(GEN_NUM, num)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// return (List) result.get(DATA); -// } -// -// @Override -// public ResourceDto createOrUpdateScriptContent(String resourceName, String content) { -// // check resource exists -// final String fullName = defaultScriptDir.concat(RESOURCE_SEPARATOR.concat(resourceName)); -// final ResourceDto parentResourceDto = getResourceDto(defaultScriptDir, -// RESOURCE_TYPE_FILE); -// if (Objects.isNull(parentResourceDto)) { -// throw new SeatunnelException( -// SeatunnelErrorEnum.NO_MATCHED_SCRIPT_SAVE_DIR, defaultScriptDir); -// } -// final ResourceDto dto = -// getResourceDto(fullName.concat(DEFAULT_FILE_SUFFIX), RESOURCE_TYPE_FILE); -// if (Objects.isNull(dto)) { -// final OnlineCreateResourceDto createDto = -// OnlineCreateResourceDto.builder() -// .type(RESOURCE_TYPE_FILE) -// .pid(parentResourceDto.getId()) -// .fileName(resourceName) -// .currentDir(defaultScriptDir) -// .suffix(RESOURCE_TYPE_FILE_SUFFIX_DEFAULT) -// .content(content) -// .build(); -// onlineCreateResource(createDto); -// return getResourceDto(fullName.concat(DEFAULT_FILE_SUFFIX), RESOURCE_TYPE_FILE); -// } else { -// updateContent(dto.getId(), content); -// return dto; -// } -// } -// -// @Override -// public PageData listTaskInstance(ListTaskInstanceDto dto) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format(QUERY_TASK_LIST_PAGING, -// defaultProjectCode))) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// PROCESS_INSTANCE_NAME, -// dto.getName(), -// PAGE_NO, -// dto.getPageNo(), -// PAGE_SIZE, -// dto.getPageSize())) -// .withToken(TOKEN, token) -// .execute(Map.class); -// -// checkResult(result, false); -// final Map map = MapUtils.getMap(result, DATA); -// final List> taskInstanceList = -// (List>) map.get(DATA_TOTAL_LIST); -// final int total = MapUtils.getIntValue(map, DATA_TOTAL); -// if (CollectionUtils.isEmpty(taskInstanceList)) { -// return PageData.empty(); -// } -// -// final List data = -// taskInstanceList.stream() -// .map(m -> this.mapToPojo(m, TaskInstanceDto.class)) -// .collect(Collectors.toList()); -// return new PageData<>(total, data); -// } -// -// @Override -// public void deleteProcessDefinition(long code) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format( -// DELETE_PROCESS_DEFINITION, -// defaultProjectCode, -// code))) -// .withMethod(Connection.Method.DELETE) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// @Override -// public PageData listProcessInstance(ListProcessInstanceDto dto) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format(PROCESS_INSTANCE_LIST, defaultProjectCode))) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// SEARCH_VAL, -// dto.getName(), -// PAGE_NO, -// dto.getPageNo(), -// PAGE_SIZE, -// dto.getPageSize())) -// .withToken(TOKEN, token) -// .execute(Map.class); -// -// final Map map = MapUtils.getMap(result, DATA); -// final List> processInstanceList = -// (List>) map.get(DATA_TOTAL_LIST); -// final int total = MapUtils.getIntValue(map, DATA_TOTAL); -// if (CollectionUtils.isEmpty(processInstanceList)) { -// return PageData.empty(); -// } -// -// final List data = -// processInstanceList.stream() -// .map(m -> this.mapToPojo(m, ProcessInstanceDto.class)) -// .collect(Collectors.toList()); -// return new PageData<>(total, data); -// } -// -// @Override -// public void killProcessInstance(long processInstanceId) { -// execute(processInstanceId, ExecuteTypeEnum.STOP); -// } -// -// @Override -// public InstanceLogDto queryInstanceLog(long instanceId) { -// -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(LOG_DETAIL)) -// .withData( -// createParamMap( -// TASK_INSTANCE_ID, -// instanceId, -// LOG_SKIP_LINE_NUM, -// LOG_SKIP_LINE_NUM_DEFAULT, -// LOG_LIMIT_NUM, -// LOG_LIMIT_NUM_DEFAULT)) -// .withMethod(Connection.Method.GET) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// final Map map = MapUtils.getMap(result, DATA); -// final String logContent = MapUtils.getString(map, LOG_MESSAGE); -// -// return InstanceLogDto.builder().logContent(logContent).build(); -// } -// -// private ProjectDto queryProjectCodeByName(String projectName) throws IOException { -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(QUERY_PROJECT_LIST_PAGING)) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// SEARCH_VAL, -// projectName, -// PAGE_NO, -// PAGE_NO_DEFAULT, -// PAGE_SIZE, -// PAGE_SIZE_DEFAULT)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// -// final List> projectList = -// (List>) MapUtils.getMap(result, DATA).get(DATA_TOTAL_LIST); -// final ProjectDto projectDto = -// projectList.stream() -// .map(m -> this.mapToPojo(m, ProjectDto.class)) -// .filter(p -> p.getName().equalsIgnoreCase(projectName)) -// .findAny() -// .orElse(null); -// if (Objects.isNull(projectDto)) { -// throw new SeatunnelException(NO_MATCHED_PROJECT, projectName); -// } -// return projectDto; -// } -// -// @Override -// public void execute(long processInstanceId, ExecuteTypeEnum executeType) { -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(String.format(EXECUTE, defaultProjectCode))) -// .withMethod(Connection.Method.POST) -// .withRequestBody(this.objectToString(null)) -// .withData( -// createParamMap( -// PROCESS_INSTANCE_ID, -// processInstanceId, -// EXECUTE_TYPE, -// executeType.name())) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// private TaskDefinitionDto buildTaskDefinitionJson( -// Long taskCode, TaskDescriptionDto taskDescriptionDto) { -// final ResourceDto resourceDto = -// createOrUpdateScriptContent( -// taskDescriptionDto.getName(), taskDescriptionDto.getContent()); -// final TaskDefinitionDto taskDefinitionDto = new TaskDefinitionDto(); -// taskDefinitionDto.setCode(taskCode); -// taskDefinitionDto.setName(taskDescriptionDto.getName()); -// taskDefinitionDto.setDescription(DESCRIPTION_DEFAULT); -// taskDefinitionDto.setTaskType(TASK_TYPE_DEFAULT); -// -// final TaskParamDto taskParamDto = new TaskParamDto(); -// -// taskParamDto.setResourceList(Collections.singletonList(resourceDto)); -// -// final List localParams = getLocalParams(taskDescriptionDto); -// taskParamDto.setLocalParams(localParams); -// taskParamDto.setRawScript(taskDescriptionDto.getExecuteScript()); -// taskParamDto.setDependence(DEPENDENCE_DEFAULT); -// -// final ConditionResult conditionResult = new ConditionResult(); -// conditionResult.setSuccessNode(SUCCESS_NODE_DEFAULT); -// conditionResult.setFailedNode(FAILED_NODE_DEFAULT); -// -// taskParamDto.setConditionResult(conditionResult); -// taskParamDto.setWaitStartTimeout(WAIT_START_TIMEOUT_DEFAULT); -// taskParamDto.setSwitchResult(SWITCH_RESULT_DEFAULT); -// -// taskDefinitionDto.setTaskParams(taskParamDto); -// taskDefinitionDto.setFlag(FLAG_DEFAULT); -// taskDefinitionDto.setTaskPriority(TASK_PRIORITY_DEFAULT); -// taskDefinitionDto.setWorkerGroup(WORKER_GROUP_DEFAULT); -// taskDefinitionDto.setFailRetryTimes(taskDescriptionDto.getRetryTimes()); -// taskDefinitionDto.setFailRetryInterval(taskDescriptionDto.getRetryInterval()); -// taskDefinitionDto.setTimeoutFlag(TIMEOUT_FLAG_DEFAULT); -// taskDefinitionDto.setTimeoutNotifyStrategy(TIMEOUT_NOTIFY_STRATEGY_DEFAULT); -// taskDefinitionDto.setTimeout(TIMEOUT_DEFAULT); -// taskDefinitionDto.setDelayTime(DELAY_TIME_DEFAULT); -// taskDefinitionDto.setEnvironmentCode(ENVIRONMENT_CODE_DEFAULT); -// taskDefinitionDto.setVersion(VERSION_DEFAULT); -// return taskDefinitionDto; -// } -// -// private List getLocalParams(TaskDescriptionDto taskDescriptionDto) { -// final Map params = taskDescriptionDto.getParams(); -// if (CollectionUtils.isEmpty(params)) { -// return Collections.emptyList(); -// } -// final List localParams = Lists.newArrayListWithCapacity(params.size()); -// params.forEach( -// (k, v) -> { -// final LocalParam localParam = new LocalParam(); -// localParam.setProp(k); -// localParam.setDirect(LOCAL_PARAMS_DIRECT_DEFAULT); -// localParam.setType(LOCAL_PARAMS_TYPE_DEFAULT); -// localParam.setValue(v); -// localParams.add(localParam); -// }); -// return localParams; -// } -// -// private List buildTaskRelationJson( -// Long taskCode, TaskDescriptionDto taskDescriptionDto) { -// -// final TaskRelationDto taskRelationDto = new TaskRelationDto(); -// taskRelationDto.setName(BLANK_SPACE); -// taskRelationDto.setPreTaskCode(PRE_TASK_CODE_DEFAULT); -// taskRelationDto.setPreTaskVersion(PRE_TASK_VERSION_DEFAULT); -// taskRelationDto.setPostTaskCode(taskCode); -// taskRelationDto.setPostTaskVersion(POST_TASK_VERSION_DEFAULT); -// taskRelationDto.setConditionType(CONDITION_TYPE_DEFAULT); -// taskRelationDto.setConditionParams(null); -// return Collections.singletonList(taskRelationDto); -// } -// -// private List buildLocation(List taskCode) { -// final List locations = Lists.newArrayListWithCapacity(taskCode.size()); -// for (int i = 0; i < taskCode.size(); i++) { -// final LocationDto locationDto = new LocationDto(); -// locationDto.setTaskCode(taskCode.get(i)); -// locationDto.setX(LOCATIONS_X_DEFAULT * i); -// locationDto.setY(LOCATIONS_Y_DEFAULT * i); -// locations.add(locationDto); -// } -// return locations; -// } -// -// private void updateContent(int id, String content) { -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(String.format(UPDATE_CONTENT, id))) -// .withMethod(Connection.Method.PUT) -// .withData( -// createParamMap( -// RESOURCE_ID, id, RESOURCE_TYPE_FILE_CONTENT, content)) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// private void onlineCreateResource(OnlineCreateResourceDto createDto) { -// final Map map = createDto.toMap(); -// final Map result = -// HttpUtils.builder() -// .withUrl(apiPrefix.concat(ONLINE_CREATE_RESOURCE)) -// .withMethod(Connection.Method.POST) -// .withData(translate(map)) -// .withRequestBody(this.objectToString(null)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// checkResult(result, false); -// } -// -// private ResourceDto getResourceDto(String fullName, String fileType) { -// final Map result = -// HttpUtils.builder() -// .withUrl( -// apiPrefix.concat( -// String.format(QUERY_RESOURCE, RESOURCE_ID_DEFAULT))) -// .withMethod(Connection.Method.GET) -// .withData( -// createParamMap( -// FULL_NAME, fullName, RESOURCE_TYPE, RESOURCE_TYPE_FILE)) -// .withToken(TOKEN, token) -// .execute(Map.class); -// final int code = checkResult(result, true); -// if (code != CODE_SUCCESS) { -// return null; -// } -// final Map map = MapUtils.getMap(result, DATA); -// return this.mapToPojo(map, ResourceDto.class); -// } -// -// private int checkResult(Map result, boolean ignore) { -// final int intValue = MapUtils.getIntValue(result, CODE, -1); -// if (!ignore && CODE_SUCCESS != intValue) { -// final String msg = MapUtils.getString(result, MSG); -// throw new SeatunnelException(UNEXPECTED_RETURN_CODE, intValue, msg); -// } -// return intValue; -// } -// -// private Map translate(Map origin) { -// final HashMap map = Maps.newHashMapWithExpectedSize(origin.size()); -// origin.forEach((k, v) -> map.put(k, String.valueOf(v))); -// return map; -// } -// -// private T mapToPojo(Map map, Class pojo) { -// try { -// return MAPPER.readValue(MAPPER.writeValueAsString(map), pojo); -// } catch (JsonProcessingException e) { -// log.error("Map translate to Pojo failed.", e); -// throw new SeatunnelException(SeatunnelErrorEnum.JSON_TRANSFORM_FAILED); -// } -// } -// -// private String objectToString(Object o) { -// try { -// return MAPPER.writeValueAsString(o); -// } catch (JsonProcessingException e) { -// log.error("Map translate to Pojo failed.", e); -// throw new SeatunnelException(SeatunnelErrorEnum.JSON_TRANSFORM_FAILED); -// } -// } -// } diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/InstanceServiceImpl.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/InstanceServiceImpl.java deleted file mode 100644 index 48a1ec65e..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/InstanceServiceImpl.java +++ /dev/null @@ -1,87 +0,0 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package org.apache.seatunnel.scheduler.dolphinscheduler.impl; -// -// import org.apache.seatunnel.scheduler.dolphinscheduler.IDolphinschedulerService; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessInstanceDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.enums.RunFrequencyEnum; -// import org.apache.seatunnel.scheduler.dolphinscheduler.utils.StatusUtils; -// import org.apache.seatunnel.server.common.PageData; -// import org.apache.seatunnel.spi.scheduler.IInstanceService; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceDto; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceListDto; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceLogDto; -// -// import org.springframework.stereotype.Component; -// -// import lombok.extern.slf4j.Slf4j; -// -// import javax.annotation.Resource; -// -// import java.util.List; -// import java.util.stream.Collectors; -// -// @Slf4j -// @Component -// public class InstanceServiceImpl implements IInstanceService { -// -// @Resource private IDolphinschedulerService iDolphinschedulerService; -// -// @Override -// public PageData list(InstanceListDto dto) { -// -// final ListProcessInstanceDto listDto = new ListProcessInstanceDto(); -// listDto.setName(dto.getName()); -// listDto.setPageNo(dto.getPageNo()); -// listDto.setPageSize(dto.getPageSize()); -// -// // use list process instance instead of list task instance. -// final PageData instancePageData = -// iDolphinschedulerService.listProcessInstance(listDto); -// -// final List data = -// instancePageData.getData().stream() -// .map( -// t -> -// InstanceDto.builder() -// .instanceId(t.getId()) -// .jobId(t.getProcessDefinitionCode()) -// .instanceName(t.getName()) -// .status( -// StatusUtils.getDolphinSchedulerStatus( -// t.getState()) -// .name()) -// .startTime(t.getStartTime()) -// .endTime(t.getEndTime()) -// .submitTime(t.getScheduleTime()) -// .executionDuration(t.getDuration()) -// .retryTimes(t.getRunTimes()) -// .runFrequency( -// RunFrequencyEnum.parse(t.getCommandType()) -// .name()) -// .build()) -// .collect(Collectors.toList()); -// return new PageData<>(instancePageData.getTotalCount(), data); -// } -// -// @Override -// public InstanceLogDto queryInstanceLog(long instanceId) { -// return iDolphinschedulerService.queryInstanceLog(instanceId); -// } -// } diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/JobServiceImpl.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/JobServiceImpl.java deleted file mode 100644 index 2c8eaffc0..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/impl/JobServiceImpl.java +++ /dev/null @@ -1,301 +0,0 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -// package org.apache.seatunnel.scheduler.dolphinscheduler.impl; -// -// import org.apache.seatunnel.scheduler.dolphinscheduler.IDolphinschedulerService; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ListProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.ProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.SchedulerDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.StartProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.TaskDescriptionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.dto.UpdateProcessDefinitionDto; -// import org.apache.seatunnel.scheduler.dolphinscheduler.enums.ReleaseStateEnum; -// import org.apache.seatunnel.server.common.DateUtils; -// import org.apache.seatunnel.server.common.PageData; -// import org.apache.seatunnel.server.common.SeatunnelErrorEnum; -// import org.apache.seatunnel.server.common.SeatunnelException; -// import org.apache.seatunnel.spi.scheduler.IInstanceService; -// import org.apache.seatunnel.spi.scheduler.IJobService; -// import org.apache.seatunnel.spi.scheduler.dto.ComplementDataDto; -// import org.apache.seatunnel.spi.scheduler.dto.ExecuteDto; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceDto; -// import org.apache.seatunnel.spi.scheduler.dto.InstanceListDto; -// import org.apache.seatunnel.spi.scheduler.dto.JobDto; -// import org.apache.seatunnel.spi.scheduler.dto.JobListDto; -// import org.apache.seatunnel.spi.scheduler.dto.JobSimpleInfoDto; -// import org.apache.seatunnel.spi.scheduler.enums.ExecuteTypeEnum; -// -// import org.springframework.beans.factory.annotation.Value; -// import org.springframework.stereotype.Component; -// import org.springframework.util.CollectionUtils; -// -// import lombok.extern.slf4j.Slf4j; -// -// import javax.annotation.Resource; -// -// import java.util.Date; -// import java.util.List; -// import java.util.Objects; -// import java.util.concurrent.CompletableFuture; -// import java.util.stream.Collectors; -// -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DEPENDENT_MODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.DRY_RUN; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.EXEC_TYPE_COMPLEMENT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.EXEC_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.FAILURE_STRATEGY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_NO_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PAGE_SIZE_MIN; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.PROCESS_INSTANCE_PRIORITY_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RELEASE_STATE_OFFLINE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RELEASE_STATE_ONLINE; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RUN_MODE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.RUN_MODE_PARALLEL; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.TASK_DEPEND_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_GROUP_ID_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WARNING_TYPE_DEFAULT; -// import static -// org.apache.seatunnel.scheduler.dolphinscheduler.constants.DolphinschedulerConstants.WORKER_GROUP_DEFAULT; -// import static org.apache.seatunnel.server.common.Constants.COMMA; -// import static org.apache.seatunnel.server.common.DateUtils.DEFAULT_DATETIME_FORMAT; -// -// @Component -// @Slf4j -// public class JobServiceImpl implements IJobService { -// @Resource private IDolphinschedulerService iDolphinschedulerService; -// -// @Resource private IInstanceService iInstanceService; -// -// @Value("${maxWaitingTimes:10}") -// private long maxWaitingTimes; -// -// @Value("${waitingSleepTime:100}") -// private long waitingSleepTime; -// -// @Override -// public long submitJob(JobDto dto) { -// // one process == one seatunnel script == one job -// -// final ProcessDefinitionDto processDefinition = getProcessDefinitionDto(dto); -// -// dto.setJobId(processDefinition.getCode()); -// iDolphinschedulerService.updateProcessDefinitionState( -// processDefinition.getCode(), processDefinition.getName(), RELEASE_STATE_ONLINE); -// final SchedulerDto schedulerDto = iDolphinschedulerService.createOrUpdateSchedule(dto); -// iDolphinschedulerService.scheduleOnline(schedulerDto.getId()); -// -// return processDefinition.getCode(); -// } -// -// @Override -// public void offlineJob(JobDto dto) { -// iDolphinschedulerService.updateProcessDefinitionState( -// dto.getJobId(), dto.getJobName(), RELEASE_STATE_OFFLINE); -// } -// -// @Override -// public PageData list(JobListDto dto) { -// final ListProcessDefinitionDto listDto = new ListProcessDefinitionDto(); -// listDto.setName(dto.getName()); -// listDto.setPageNo(dto.getPageNo()); -// listDto.setPageSize(dto.getPageSize()); -// -// final PageData processPageData = -// iDolphinschedulerService.listProcessDefinition(listDto); -// final List data = -// processPageData.getData().stream() -// .map( -// p -> -// JobSimpleInfoDto.builder() -// .jobId(p.getCode()) -// .jobName(p.getName()) -// .jobStatus(p.getReleaseState()) -// .publish( -// -// ReleaseStateEnum.parse(p.getReleaseState()) -// .isPublish()) -// .createTime(p.getCreateTime()) -// .updateTime(p.getUpdateTime()) -// .creatorName(p.getUserName()) -// .menderName(p.getModifyBy()) -// .build()) -// .collect(Collectors.toList()); -// return new PageData<>(processPageData.getTotalCount(), data); -// } -// -// @Override -// @SuppressWarnings("magicnumber") -// public InstanceDto execute(ExecuteDto dto) { -// -// if (dto.getExecuteTypeEnum() == ExecuteTypeEnum.RERUN) { -// iDolphinschedulerService.execute( -// dto.getJobDto().getJobId(), -// -// org.apache.seatunnel.scheduler.dolphinscheduler.ExecuteTypeEnum.REPEAT_RUNNING); -// return null; -// } -// -// ProcessDefinitionDto processDefinition = null; -// final JobDto jobDto = dto.getJobDto(); -// if (Objects.isNull(jobDto.getJobId())) { -// // need to create a temporary process definition and execute it. -// processDefinition = getProcessDefinitionDto(jobDto); -// jobDto.setJobId(processDefinition.getCode()); -// iDolphinschedulerService.updateProcessDefinitionState( -// processDefinition.getCode(), processDefinition.getName(), -// RELEASE_STATE_ONLINE); -// } -// -// final ComplementDataDto complementDataDto = dto.getComplementDataDto(); -// String execType; -// String runMode = RUN_MODE_DEFAULT; -// -// Date startTime = new Date(); -// Date endTime = new Date(); -// int parallelismNum = 1; -// if (Objects.isNull(complementDataDto)) { -// execType = EXEC_TYPE_DEFAULT; -// } else { -// execType = EXEC_TYPE_COMPLEMENT; -// if (Objects.nonNull(complementDataDto.getParallelismNum())) { -// runMode = RUN_MODE_PARALLEL; -// } -// startTime = complementDataDto.getStartTime(); -// endTime = complementDataDto.getEndTime(); -// parallelismNum = complementDataDto.getParallelismNum(); -// } -// -// final StartProcessDefinitionDto startProcessDefinitionDto = -// StartProcessDefinitionDto.builder() -// .processDefinitionCode(jobDto.getJobId()) -// .failureStrategy(FAILURE_STRATEGY_DEFAULT) -// .warningType(WARNING_TYPE_DEFAULT) -// .warningGroupId(WARNING_GROUP_ID_DEFAULT) -// .taskDependType(TASK_DEPEND_TYPE_DEFAULT) -// .runMode(runMode) -// .processInstancePriority(PROCESS_INSTANCE_PRIORITY_DEFAULT) -// .workerGroup(WORKER_GROUP_DEFAULT) -// .dryRun(DRY_RUN) -// .scheduleTime( -// DateUtils.format(startTime, DEFAULT_DATETIME_FORMAT) -// .concat(COMMA) -// .concat(DateUtils.format(endTime, -// DEFAULT_DATETIME_FORMAT))) -// .execType(execType) -// .dependentMode(DEPENDENT_MODE_DEFAULT) -// .expectedParallelismNumber(parallelismNum) -// .build(); -// iDolphinschedulerService.startProcessDefinition(startProcessDefinitionDto); -// -// if (Objects.nonNull(processDefinition)) { -// -// final long code = processDefinition.getCode(); -// final String name = processDefinition.getName(); -// -// InstanceDto instanceDto = null; -// // waiting dolphinscheduler generate instance. -// for (int i = 0; i < maxWaitingTimes; i++) { -// // get instance by process definition name -// final InstanceListDto instanceListDto = -// InstanceListDto.builder() -// .pageNo(PAGE_NO_DEFAULT) -// .pageSize(PAGE_SIZE_MIN) -// .name(processDefinition.getName()) -// .build(); -// final PageData instancePageData = -// iInstanceService.list(instanceListDto); -// if (!CollectionUtils.isEmpty(instancePageData.getData())) { -// instanceDto = instancePageData.getData().get(0); -// break; -// } -// try { -// Thread.sleep(waitingSleepTime); -// } catch (InterruptedException e) { -// log.error("waiting for getting instance failed", e); -// throw new SeatunnelException(SeatunnelErrorEnum.GET_INSTANCE_FAILED); -// } -// } -// -// CompletableFuture.runAsync( -// () -> { -// // offline temporary process definition -// iDolphinschedulerService.updateProcessDefinitionState( -// code, name, RELEASE_STATE_OFFLINE); -// }) -// .whenComplete( -// (_return, e) -> { -// if (Objects.nonNull(e)) { -// log.error( -// "clear temporary process definition failed, process -// definition code is [{}], name is [{}]", -// code, -// name, -// e); -// } -// }); -// return instanceDto; -// -// } else { -// return null; -// } -// } -// -// @Override -// public void kill(Long instanceId) { -// iDolphinschedulerService.killProcessInstance(instanceId); -// } -// -// private ProcessDefinitionDto getProcessDefinitionDto(JobDto dto) { -// final TaskDescriptionDto taskDescriptionDto = -// TaskDescriptionDto.builder() -// .name(dto.getJobName()) -// .executeScript(dto.getExecutorScript()) -// .content(dto.getJobContent()) -// .params(dto.getParams()) -// .retryInterval(dto.getSchedulerConfigDto().getRetryInterval()) -// .retryTimes(dto.getSchedulerConfigDto().getRetryTimes()) -// .build(); -// -// final UpdateProcessDefinitionDto processDto = -// UpdateProcessDefinitionDto.builder() -// .name(dto.getJobName()) -// .startTime(dto.getSchedulerConfigDto().getStartTime()) -// .endTime(dto.getSchedulerConfigDto().getEndTime()) -// .cronExpression(dto.getSchedulerConfigDto().getTriggerExpression()) -// .taskDescriptionDto(taskDescriptionDto) -// .processDefinitionCode(dto.getJobId()) -// .build(); -// -// return iDolphinschedulerService.createOrUpdateProcessDefinition(processDto); -// } -// } diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/HttpUtils.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/HttpUtils.java deleted file mode 100644 index a113006e6..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/HttpUtils.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.utils; - -import org.apache.seatunnel.server.common.SeatunnelException; - -import org.jsoup.Connection; -import org.jsoup.Jsoup; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkState; -import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.HTTP_REQUEST_FAILED; - -@Slf4j -public class HttpUtils { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - @SuppressWarnings("MagicNumber") - public static Map createParamMap(Object... elements) { - if (elements == null || elements.length == 0 || elements.length % 2 == 1) { - throw new IllegalArgumentException("params length must be even!"); - } - Map paramMap = Maps.newHashMapWithExpectedSize(elements.length / 2); - for (int i = 0; i < elements.length / 2; i++) { - Object key = elements[2 * i]; - Object value = elements[2 * i + 1]; - if (key == null) { - continue; - } - - paramMap.put(key.toString(), value == null ? "" : value.toString()); - } - - return paramMap; - } - - public static Builder builder() { - return new Builder(); - } - - @SuppressWarnings("MagicNumber") - public static class Builder { - private Connection connection; - private String url; - private Optional> data = Optional.empty(); - private Optional requestBody = Optional.empty(); - private Optional maxBodySize = Optional.of(2147483647); - private Optional postDataCharset = Optional.of("UTF-8"); - private Optional ignoreContentType = Optional.of(true); - private Optional timeout = Optional.of(30000); - private Optional> headers; - private Optional method = Optional.empty(); - - private Optional type; - - public Builder() { - HashMap defaultMap = Maps.newHashMap(); - defaultMap.put("Content-Type", "application/json; charset=UTF-8"); - headers = Optional.of(defaultMap); - } - - public Builder withUrl(String url) { - this.url = url; - return this; - } - - public Builder withData(Map data) { - this.data = Optional.ofNullable(data); - return this; - } - - public Builder withRequestBody(String requestBody) { - this.requestBody = Optional.ofNullable(requestBody); - return this; - } - - public Builder withMaxBodySize(int maxBodySize) { - this.maxBodySize = Optional.of(maxBodySize); - return this; - } - - public Builder withPostDataCharset(String postDataCharset) { - this.postDataCharset = Optional.ofNullable(postDataCharset); - return this; - } - - public Builder withIgnoreContentType(boolean ignoreContentType) { - this.ignoreContentType = Optional.of(ignoreContentType); - return this; - } - - public Builder withTimeout(int timeout) { - this.timeout = Optional.of(timeout); - return this; - } - - public Builder withHeaders(Map headers) { - this.headers = Optional.ofNullable(headers); - return this; - } - - public Builder withMethod(Connection.Method method) { - this.method = Optional.ofNullable(method); - return this; - } - - public Builder withToken(String tokenKey, String tokenValue) { - this.headers.ifPresent(map -> map.put(tokenKey, tokenValue)); - return this; - } - - private Connection build() { - checkState(!Strings.isNullOrEmpty(url), "request url is empty"); - connection = Jsoup.connect(url); - data.ifPresent(connection::data); - requestBody.ifPresent(connection::requestBody); - maxBodySize.ifPresent(connection::maxBodySize); - postDataCharset.ifPresent(connection::postDataCharset); - ignoreContentType.ifPresent(connection::ignoreContentType); - timeout.ifPresent(connection::timeout); - headers.ifPresent(connection::headers); - method.ifPresent(connection::method); - - return connection; - } - - public T execute(Class type) { - this.build(); - try { - Connection.Response response = connection.execute(); - return MAPPER.readValue(response.body(), type); - } catch (IOException e) { - log.error("Request url {} failed", this.url, e); - throw new SeatunnelException(HTTP_REQUEST_FAILED, this.url); - } - } - - public void execute() throws IOException { - this.build(); - try { - connection.execute(); - } catch (IOException e) { - log.error("Request url {} failed", this.url, e); - throw new SeatunnelException(HTTP_REQUEST_FAILED, this.url); - } - } - } -} diff --git a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/StatusUtils.java b/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/StatusUtils.java deleted file mode 100644 index 1568e5e74..000000000 --- a/seatunnel-server/seatunnel-scheduler/seatunnel-scheduler-dolphinscheduler/src/main/java/org/apache/seatunnel/scheduler/dolphinscheduler/utils/StatusUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.scheduler.dolphinscheduler.utils; - -import org.apache.seatunnel.spi.scheduler.enums.InstanceStatusEnum; - -public class StatusUtils { - - public static InstanceStatusEnum getDolphinSchedulerStatus(String status) { - switch (status) { - case "SUBMITTED_SUCCESS": - case "RUNNING_EXECUTION": - case "DISPATCH": - return InstanceStatusEnum.RUNNING; - case "STOP": - case "KILL": - return InstanceStatusEnum.STOPPED; - case "FAILURE": - return InstanceStatusEnum.FAILED; - case "FORCED_SUCCESS": - case "SUCCESS": - return InstanceStatusEnum.SUCCESS; - default: - return InstanceStatusEnum.UNKNOWN; - } - } -} diff --git a/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java b/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java index 9f1bffa27..d6636240c 100644 --- a/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java +++ b/seatunnel-server/seatunnel-server-common/src/main/java/org/apache/seatunnel/server/common/SeatunnelErrorEnum.java @@ -122,8 +122,8 @@ public enum SeatunnelErrorEnum { "can not found connector for datasource [%s]"), DATA_SOURCE_HAD_USED( 1600000, - "data source already used ( workflowName - taskName):{0}", - "datasource is using:{0}"), + "datasource can not be delete because it used by task", + "datasource can not be delete because it used by task"), INVALID_DATASOURCE(-70001, "Datasource [{0}] invalid", "datasource [{0}] invalid"), MISSING_PARAM(1777000, "param miss [{0}]", "param miss [{0}]"), ; diff --git a/seatunnel-server/seatunnel-spi/pom.xml b/seatunnel-server/seatunnel-spi/pom.xml deleted file mode 100644 index a9658bb50..000000000 --- a/seatunnel-server/seatunnel-spi/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-server - ${revision} - - - seatunnel-spi - - - 1.8 - - - - - org.apache.seatunnel - seatunnel-server-common - ${project.version} - - - - diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IInstanceService.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IInstanceService.java deleted file mode 100644 index dad0ee4e9..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IInstanceService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler; - -import org.apache.seatunnel.server.common.PageData; -import org.apache.seatunnel.spi.scheduler.dto.InstanceDto; -import org.apache.seatunnel.spi.scheduler.dto.InstanceListDto; -import org.apache.seatunnel.spi.scheduler.dto.InstanceLogDto; - -public interface IInstanceService { - PageData list(InstanceListDto dto); - - InstanceLogDto queryInstanceLog(long instanceId); -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IJobService.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IJobService.java deleted file mode 100644 index 7a6291240..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/IJobService.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler; - -import org.apache.seatunnel.server.common.PageData; -import org.apache.seatunnel.spi.scheduler.dto.ExecuteDto; -import org.apache.seatunnel.spi.scheduler.dto.InstanceDto; -import org.apache.seatunnel.spi.scheduler.dto.JobDto; -import org.apache.seatunnel.spi.scheduler.dto.JobListDto; -import org.apache.seatunnel.spi.scheduler.dto.JobSimpleInfoDto; - -public interface IJobService { - - long submitJob(JobDto dto); - - void offlineJob(JobDto dto); - - PageData list(JobListDto dto); - - InstanceDto execute(ExecuteDto dto); - - void kill(Long instanceId); -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/constants/SchedulerConstant.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/constants/SchedulerConstant.java deleted file mode 100644 index 1d8d030f5..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/constants/SchedulerConstant.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.constants; - -public class SchedulerConstant { - public static final int RETRY_INTERVAL_DEFAULT = 0; - public static final int RETRY_TIMES_DEFAULT = 0; - public static final String NEVER_TRIGGER_EXPRESSION = "* * * ? * * 1970"; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceDto.java deleted file mode 100644 index 56deb9ded..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceDto.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.Date; - -@Data -@Builder -public class InstanceDto { - private long instanceId; - private long jobId; - private String instanceName; - private Date submitTime; - private Date startTime; - private Date endTime; - private String status; - private String executionDuration; - private long retryTimes; - private String runFrequency; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceListDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceListDto.java deleted file mode 100644 index 0b05302d3..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceListDto.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -@Builder -@Data -public class InstanceListDto { - private String name; - private int pageNo; - private int pageSize; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceLogDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceLogDto.java deleted file mode 100644 index 9a16b112b..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/InstanceLogDto.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -@Builder -@Data -public class InstanceLogDto { - private String logContent; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobDto.java deleted file mode 100644 index 3f30194b6..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobDto.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.Map; - -@Data -@Builder -public class JobDto { - private Long jobId; - private String jobName; - private String executorScript; - private String jobContent; - private Integer operatorId; - private Map params; - private SchedulerConfigDto schedulerConfigDto; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobListDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobListDto.java deleted file mode 100644 index 5e5b1fcdc..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobListDto.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -public class JobListDto { - private String name; - private Integer pageNo; - private Integer pageSize; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobSimpleInfoDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobSimpleInfoDto.java deleted file mode 100644 index 85e0bccd2..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/JobSimpleInfoDto.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.Date; - -@Builder -@Data -public class JobSimpleInfoDto { - private long jobId; - private String jobName; - private String jobStatus; - private Boolean publish; - private String creatorName; - private String menderName; - private Date createTime; - private Date updateTime; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/SchedulerConfigDto.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/SchedulerConfigDto.java deleted file mode 100644 index aa71ac686..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/dto/SchedulerConfigDto.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.Date; - -@Data -@Builder -public class SchedulerConfigDto { - private int retryTimes; - private int retryInterval; - private Date startTime; - private Date endTime; - private String triggerExpression; -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/ExecuteTypeEnum.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/ExecuteTypeEnum.java deleted file mode 100644 index ca266dc90..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/ExecuteTypeEnum.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.enums; - -import org.apache.seatunnel.server.common.SeatunnelException; - -import static org.apache.seatunnel.server.common.SeatunnelErrorEnum.NO_SUCH_ELEMENT; - -public enum ExecuteTypeEnum { - TEMPORARY, - MANUAL, - SCHEDULER, - RERUN, - ; - - public static ExecuteTypeEnum parse(int executeType) { - for (ExecuteTypeEnum value : values()) { - if (value.ordinal() == executeType) { - return value; - } - } - throw new SeatunnelException(NO_SUCH_ELEMENT); - } -} diff --git a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/InstanceStatusEnum.java b/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/InstanceStatusEnum.java deleted file mode 100644 index 10fd24c52..000000000 --- a/seatunnel-server/seatunnel-spi/src/main/java/org/apache/seatunnel/spi/scheduler/enums/InstanceStatusEnum.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.spi.scheduler.enums; - -public enum InstanceStatusEnum { - RUNNING, - STOPPED, - FAILED, - SUCCESS, - UNKNOWN, - ; -} diff --git a/seatunnel-ui/package.json b/seatunnel-ui/package.json index c0ae6853e..7df06eded 100644 --- a/seatunnel-ui/package.json +++ b/seatunnel-ui/package.json @@ -27,7 +27,6 @@ "postcss": "^8.4.21", "screenfull": "6.0.1", "tailwindcss": "^3.2.7", - "vfonts": "^0.0.3", "vue": "^3.2.47", "vue-i18n": "^9.2.2", "vue-router": "^4.1.6" diff --git a/seatunnel-ui/src/common/table.ts b/seatunnel-ui/src/common/table.ts index b3e5e63dd..2f651ff8e 100644 --- a/seatunnel-ui/src/common/table.ts +++ b/seatunnel-ui/src/common/table.ts @@ -18,7 +18,7 @@ import { useSettingStore } from '@/store/setting' const getTableColumn = (data: Array<{ key: string; title: string }>) => { - const tableColumn = [] + const tableColumn : any[] = [] const settingStore = useSettingStore() settingStore.getSequenceColumn && @@ -30,7 +30,7 @@ const getTableColumn = (data: Array<{ key: string; title: string }>) => { settingStore.getDataUniqueValue && tableColumn.push( - ...data.map((i) => { + ...data.map((i: {key: any, title:any}): {key: any, title:any} => { return { title: i.title, key: i.key diff --git a/seatunnel-ui/src/layouts/dashboard/header/user/use-user-dropdown.ts b/seatunnel-ui/src/layouts/dashboard/header/user/use-user-dropdown.ts index 7611ba352..30b24afb5 100644 --- a/seatunnel-ui/src/layouts/dashboard/header/user/use-user-dropdown.ts +++ b/seatunnel-ui/src/layouts/dashboard/header/user/use-user-dropdown.ts @@ -26,7 +26,7 @@ export function useUserDropdown() { const handleSelect = (key: string) => { if (key === 'help') { - window.open('http://seatunnel.incubator.apache.org/versions/') + window.open('http://seatunnel.apache.org/versions/') } else if (key === 'setting') { router.push({ path: '/setting' }) } else if (key === 'logout') { diff --git a/seatunnel-ui/src/locales/en_US/data-pipes.ts b/seatunnel-ui/src/locales/en_US/data-pipes.ts deleted file mode 100644 index c8d549f6b..000000000 --- a/seatunnel-ui/src/locales/en_US/data-pipes.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export default { - id: 'Id', - data_pipes: 'Data Pipes', - create: 'Create', - name: 'Name', - state: 'State', - create_time: 'Create Time', - update_time: 'Update Time', - operation: 'Operation', - un_start: 'Un Start', - execute: 'Execute', - edit: 'Edit', - publish: 'Publish', - delete: 'Delete', - save: 'Save', - cancel: 'Cancel', - script: 'Script', - kill: 'Kill', - stop: 'Stop', - add: 'Add', - key: 'Key', - value: 'Value', - name_tips: 'Required fields,number, letter case, 100 characters', - data_pipes_delete_tips: - 'Whether to delete the data Pipe，It cannot be restored after being deleted', - data_pipes_publish_tips: 'Whether to Publish the data pipe', - model_validate_tips: 'Required Fields' -} diff --git a/seatunnel-ui/src/locales/en_US/datasource.ts b/seatunnel-ui/src/locales/en_US/datasource.ts index e811af285..27b2412c6 100644 --- a/seatunnel-ui/src/locales/en_US/datasource.ts +++ b/seatunnel-ui/src/locales/en_US/datasource.ts @@ -35,6 +35,7 @@ export default { operation: 'Operation', click_to_view: 'Click to view', delete: 'Delete', + delete_confirm: 'Are you sure delete ?', confirm: 'Confirm', cancel: 'Cancel', create: 'Create', diff --git a/seatunnel-ui/src/locales/en_US/index.ts b/seatunnel-ui/src/locales/en_US/index.ts index 0cd9fb6a4..4c630c18b 100644 --- a/seatunnel-ui/src/locales/en_US/index.ts +++ b/seatunnel-ui/src/locales/en_US/index.ts @@ -19,9 +19,7 @@ import login from '@/locales/en_US/login' import menu from '@/locales/en_US/menu' import modal from '@/locales/en_US/modal' import user_manage from '@/locales/en_US/user-manage' -import data_pipes from '@/locales/en_US/data-pipes' import log from '@/locales/en_US/log' -import jobs from '@/locales/en_US/jobs' import tasks from '@/locales/en_US/tasks' import setting from '@/locales/en_US/setting' import datasource from '@/locales/en_US/datasource' @@ -39,9 +37,7 @@ export default { menu, modal, user_manage, - data_pipes, log, - jobs, tasks, setting, datasource, diff --git a/seatunnel-ui/src/locales/en_US/jobs.ts b/seatunnel-ui/src/locales/en_US/jobs.ts deleted file mode 100644 index f7f53f4aa..000000000 --- a/seatunnel-ui/src/locales/en_US/jobs.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export default { - script_id: 'Script Id', - job_id: 'Job Id', - jobs: 'Jobs', - search: 'Search', - data_pipe_name: 'Data Pipe Name', - plan: 'Plan', - create_time: 'Create Time', - publish: 'Publish', - operation: 'Operation', - execute: 'Execute', - recycle: 'Recycle' -} diff --git a/seatunnel-ui/src/locales/en_US/menu.ts b/seatunnel-ui/src/locales/en_US/menu.ts index 358efa434..6a87b2b0c 100644 --- a/seatunnel-ui/src/locales/en_US/menu.ts +++ b/seatunnel-ui/src/locales/en_US/menu.ts @@ -16,8 +16,6 @@ */ export default { - data_pipes: 'Data Pipes', - jobs: 'Jobs', user_manage: 'User Manage', help: 'Help', setting: 'Setting', diff --git a/seatunnel-ui/src/locales/zh_CN/datasource.ts b/seatunnel-ui/src/locales/zh_CN/datasource.ts index f8321b5f9..6222fd05b 100644 --- a/seatunnel-ui/src/locales/zh_CN/datasource.ts +++ b/seatunnel-ui/src/locales/zh_CN/datasource.ts @@ -35,6 +35,7 @@ export default { operation: '操作', click_to_view: '点击查看', delete: '删除', + delete_confirm: '确认删除？', confirm: '确定', cancel: '取消', create: '创建', diff --git a/seatunnel-ui/src/locales/zh_CN/index.ts b/seatunnel-ui/src/locales/zh_CN/index.ts index 3f759863a..5379838c2 100644 --- a/seatunnel-ui/src/locales/zh_CN/index.ts +++ b/seatunnel-ui/src/locales/zh_CN/index.ts @@ -19,9 +19,7 @@ import login from '@/locales/zh_CN/login' import menu from '@/locales/zh_CN/menu' import modal from '@/locales/zh_CN/modal' import user_manage from '@/locales/zh_CN/user-manage' -import data_pipes from '@/locales/zh_CN/data-pipes' import log from '@/locales/zh_CN/log' -import jobs from '@/locales/zh_CN/jobs' import tasks from '@/locales/zh_CN/tasks' import setting from '@/locales/zh_CN/setting' import datasource from '@/locales/zh_CN/datasource' @@ -39,9 +37,7 @@ export default { menu, modal, user_manage, - data_pipes, log, - jobs, tasks, setting, datasource, diff --git a/seatunnel-ui/src/locales/zh_CN/jobs.ts b/seatunnel-ui/src/locales/zh_CN/jobs.ts deleted file mode 100644 index 94cd79f05..000000000 --- a/seatunnel-ui/src/locales/zh_CN/jobs.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export default { - script_id: '脚本 Id', - job_id: '工作 Id', - jobs: '工作', - search: '搜索', - data_pipe_name: '数据管道名称', - plan: '计划', - create_time: '创建时间', - publish: '发布', - operation: '操作', - execute: '执行', - recycle: '回收' -} diff --git a/seatunnel-ui/src/locales/zh_CN/menu.ts b/seatunnel-ui/src/locales/zh_CN/menu.ts index 073585413..36a9b5a49 100644 --- a/seatunnel-ui/src/locales/zh_CN/menu.ts +++ b/seatunnel-ui/src/locales/zh_CN/menu.ts @@ -16,8 +16,6 @@ */ export default { - data_pipes: '数据管道', - jobs: '工作', user_manage: '用户管理', help: '帮助', setting: '设置', diff --git a/seatunnel-ui/src/router/data-pipes.ts b/seatunnel-ui/src/router/data-pipes.ts deleted file mode 100644 index 4065c7a01..000000000 --- a/seatunnel-ui/src/router/data-pipes.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from '@/utils' -import type { Component } from 'vue' - -const modules = import.meta.glob('/src/views/**/**.tsx') -const components: { [key: string]: Component } = utils.mapping(modules) - -export default { - path: '/data-pipes', - name: 'data-pipes', - meta: { - title: 'data-pipes' - }, - redirect: { name: 'data-pipes-list' }, - component: () => import('@/layouts/dashboard'), - children: [ - { - path: '/data-pipes/list', - name: 'data-pipes-list', - component: components['data-pipes-list'], - meta: { - title: 'data-pipes-list', - activeMenu: 'data-pipes' - } - }, - { - path: '/data-pipes/:dataPipeId', - name: 'data-pipes-detail', - component: components['data-pipes-detail'], - meta: { - title: 'data-pipes-detail', - activeMenu: 'data-pipes' - } - }, - { - path: '/data-pipes/:dataPipeId/edit', - name: 'data-pipes-edit', - component: components['data-pipes-edit'], - meta: { - title: 'data-pipes-edit', - activeMenu: 'data-pipes' - } - }, - { - path: '/data-pipes/create', - name: 'data-pipes-create', - component: components['data-pipes-create'], - meta: { - title: 'data-pipes-create', - activeMenu: 'data-pipes' - } - } - ] -} diff --git a/seatunnel-ui/src/router/index.ts b/seatunnel-ui/src/router/index.ts index fba0d6b97..2a88a6940 100644 --- a/seatunnel-ui/src/router/index.ts +++ b/seatunnel-ui/src/router/index.ts @@ -15,13 +15,13 @@ * limitations under the License. */ -import { createRouter, createWebHistory } from 'vue-router' +import { createRouter, createWebHashHistory } from 'vue-router' import routes from './routes' import NProgress from 'nprogress' import 'nprogress/nprogress.css' const router = createRouter({ - history: createWebHistory( + history: createWebHashHistory( import.meta.env.MODE === 'production' ? '/ui/' : '/' ), routes @@ -31,4 +31,4 @@ router.afterEach(() => { NProgress.done() }) -export default router +export default router \ No newline at end of file diff --git a/seatunnel-ui/src/router/jobs.ts b/seatunnel-ui/src/router/jobs.ts deleted file mode 100644 index 1f210ff35..000000000 --- a/seatunnel-ui/src/router/jobs.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import utils from '@/utils' -import type { Component } from 'vue' - -const modules = import.meta.glob('/src/views/**/**.tsx') -const components: { [key: string]: Component } = utils.mapping(modules) - -export default { - path: '/jobs', - name: 'jobs', - meta: { - title: 'jobs' - }, - redirect: { name: 'jobs-list' }, - component: () => import('@/layouts/dashboard'), - children: [ - { - path: '/jobs/list', - name: 'jobs-list', - component: components['jobs-list'], - meta: { - title: 'jobs-list', - activeMenu: 'job' - } - } - ] -} diff --git a/seatunnel-ui/src/router/routes.ts b/seatunnel-ui/src/router/routes.ts index 121ba1ec2..74acedc5e 100644 --- a/seatunnel-ui/src/router/routes.ts +++ b/seatunnel-ui/src/router/routes.ts @@ -16,8 +16,6 @@ */ import utils from '@/utils' -import dataPipes from '@/router/data-pipes' -import jobs from '@/router/jobs' import tasks from '@/router/tasks' import userManage from '@/router/user-manage' import datasource from '@/router/datasource' @@ -33,8 +31,6 @@ const basePage: RouteRecordRaw[] = [ path: '/', redirect: { name: 'login' } }, - dataPipes, - jobs, tasks, userManage, datasource, diff --git a/seatunnel-ui/src/router/sync-task.ts b/seatunnel-ui/src/router/sync-task.ts index 910e18773..4cb00f269 100644 --- a/seatunnel-ui/src/router/sync-task.ts +++ b/seatunnel-ui/src/router/sync-task.ts @@ -35,7 +35,7 @@ export default { name: 'synchronization-definition', component: components['projects-task-synchronization-definition'], meta: { - title: '同步任务定义', + title: 'synchronization-definition', activeMenu: 'projects', showSide: true // auth: 'project:seatunnel-task:view' @@ -46,7 +46,7 @@ export default { name: 'synchronization-definition-dag', component: components['projects-task-synchronization-definition-dag'], meta: { - title: '同步任务定义画布', + title: 'synchronization-definition-dag', activeMenu: 'projects', activeSide: '/task/synchronization-definition', showSide: true, @@ -58,7 +58,7 @@ export default { name: 'synchronization-instance', component: components['projects-task-synchronization-instance'], meta: { - title: '同步任务实例', + title: 'synchronization-instance', activeMenu: 'projects', showSide: true // auth: 'project:seatunnel-task-instance:view' @@ -69,7 +69,7 @@ export default { name: 'synchronization-instance-detail', component: components['projects-task-synchronization-instance-detail'], meta: { - title: '同步任务实例详情', + title: 'synchronization-instance-detail', activeMenu: 'projects', activeSide: '/task/synchronization-instance', showSide: true, diff --git a/seatunnel-ui/src/router/virtual-tables.ts b/seatunnel-ui/src/router/virtual-tables.ts index a097f2c0b..82378424c 100644 --- a/seatunnel-ui/src/router/virtual-tables.ts +++ b/seatunnel-ui/src/router/virtual-tables.ts @@ -44,7 +44,7 @@ export default { name: 'virtual-tables-create', component: components['virtual-tables-detail'], meta: { - title: '虚拟表创建', + title: 'virtual-tables-create', activeMenu: 'virtual-tables', } }, @@ -53,7 +53,7 @@ export default { name: 'virtual-tables-editor', component: components['virtual-tables-detail'], meta: { - title: '虚拟表编辑', + title: 'virtual-tables-editor', activeMenu: 'virtual-tables', } } diff --git a/seatunnel-ui/src/service/sync-task-definition/index.ts b/seatunnel-ui/src/service/sync-task-definition/index.ts index ca970265f..3abcc37aa 100644 --- a/seatunnel-ui/src/service/sync-task-definition/index.ts +++ b/seatunnel-ui/src/service/sync-task-definition/index.ts @@ -145,7 +145,6 @@ export function queryTaskDetail(jobCode: string, taskCode: string): any { }) } -// source类型，获取源名称 export function listSourceName( jobId: string, sceneMode: string, @@ -161,7 +160,7 @@ export function listSourceName( } }) } -// sink类型，获取源名称 + export function findSink( jobId: string, status: 'DOWNLOADED' | 'NOT_DOWNLOAD' | 'ALL' = 'DOWNLOADED' diff --git a/seatunnel-ui/src/views/data-pipes/components/log/index.tsx b/seatunnel-ui/src/views/data-pipes/components/log/index.tsx deleted file mode 100644 index db6ceb212..000000000 --- a/seatunnel-ui/src/views/data-pipes/components/log/index.tsx +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent } from 'vue' -import { NButton, NInput, NInputGroup, NLog, NSpace } from 'naive-ui' -import { useI18n } from 'vue-i18n' - -const Log = defineComponent({ - setup() { - const { t } = useI18n() - - return { t } - }, - render() { - return ( - - - - {this.t('log.search')} - - - - ) - } -}) - -export default Log diff --git a/seatunnel-ui/src/views/data-pipes/create/index.tsx b/seatunnel-ui/src/views/data-pipes/create/index.tsx deleted file mode 100644 index 98ce5b126..000000000 --- a/seatunnel-ui/src/views/data-pipes/create/index.tsx +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent, reactive, toRefs } from 'vue' -import { useI18n } from 'vue-i18n' -import { useRouter } from 'vue-router' -import { - NBreadcrumb, - NBreadcrumbItem, - NButton, - NCard, - NIcon, - NInput, - NSpace, - NTooltip -} from 'naive-ui' -import { BulbOutlined } from '@vicons/antd' -import { scriptAdd } from '@/service/script' -import MonacoEditor from '@/components/monaco-editor' -import type { Router } from 'vue-router' - -const DataPipesCreate = defineComponent({ - setup() { - const { t } = useI18n() - const router: Router = useRouter() - const variables = reactive({ - name: '', - type: 0, - content: '' - }) - - const handleClickDataPipes = () => { - router.push({ path: '/data-pipes/list' }) - } - - const handleAdd = () => { - scriptAdd(variables).then(() => { - handleClickDataPipes() - }) - } - - return { - t, - ...toRefs(variables), - handleClickDataPipes, - handleAdd - } - }, - render() { - return ( - - - {{ - header: () => ( - - - - {this.t('data_pipes.data_pipes')} - - - {this.t('data_pipes.create')} - - - - ), - 'header-extra': () => ( - - - {this.t('data_pipes.cancel')} - - - {this.t('data_pipes.save')} - - - ) - }} - - - - {this.t('data_pipes.name')} - - - - {{ - default: () => {this.t('data_pipes.name_tips')}, - trigger: () => ( - - - - ) - }} - - - - - - - - - ) - } -}) - -export default DataPipesCreate diff --git a/seatunnel-ui/src/views/data-pipes/detail/index.tsx b/seatunnel-ui/src/views/data-pipes/detail/index.tsx deleted file mode 100644 index 64f747f3c..000000000 --- a/seatunnel-ui/src/views/data-pipes/detail/index.tsx +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent, onMounted, reactive, toRefs } from 'vue' -import { NSpace, NCard, NBreadcrumb, NBreadcrumbItem } from 'naive-ui' -import { useI18n } from 'vue-i18n' -import { useRoute, useRouter } from 'vue-router' -import { scriptDetail } from '@/service/script' -import MonacoEditor from '@/components/monaco-editor' -import type { Router, RouteLocationNormalizedLoaded } from 'vue-router' -import type { ScriptDetail } from '@/service/script/types' - -const DataPipesDetail = defineComponent({ - setup() { - const { t } = useI18n() - const router: Router = useRouter() - const route: RouteLocationNormalizedLoaded = useRoute() - const variables = reactive({ - name: '', - type: 0, - content: '' - }) - - const handleClickDataPipes = () => { - router.push({ path: '/data-pipes/list' }) - } - - onMounted(() => { - scriptDetail(Number(route.params.dataPipeId)).then( - (res: ScriptDetail) => { - variables.name = res.name - variables.type = res.type - variables.content = res.content - } - ) - }) - - return { t, ...toRefs(variables), handleClickDataPipes } - }, - render() { - return ( - - - {{ - header: () => ( - - - - {this.t('data_pipes.data_pipes')} - - {this.name} - - - ) - }} - - - - - - ) - } -}) - -export default DataPipesDetail diff --git a/seatunnel-ui/src/views/data-pipes/edit/index.tsx b/seatunnel-ui/src/views/data-pipes/edit/index.tsx deleted file mode 100644 index ddc3282ce..000000000 --- a/seatunnel-ui/src/views/data-pipes/edit/index.tsx +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent, onMounted, reactive, toRefs } from 'vue' -import { - NBreadcrumb, - NBreadcrumbItem, - NButton, - NCard, - NSpace, - NInput, - NIcon, - NTooltip -} from 'naive-ui' -import { useI18n } from 'vue-i18n' -import { useRouter, useRoute } from 'vue-router' -import { BulbOutlined } from '@vicons/antd' -import { scriptDetail, scriptUpdate } from '@/service/script' -import MonacoEditor from '@/components/monaco-editor' -import type { Router, RouteLocationNormalizedLoaded } from 'vue-router' -import type { ScriptDetail } from '@/service/script/types' - -const DataPipesEdit = defineComponent({ - setup() { - const { t } = useI18n() - const router: Router = useRouter() - const route: RouteLocationNormalizedLoaded = useRoute() - const variables = reactive({ - name: '', - type: 0, - content: '' - }) - - const handleClickDataPipes = () => { - router.push({ path: '/data-pipes/list' }) - } - - const handleAdd = () => { - scriptUpdate(Number(route.params.dataPipeId), variables.content).then( - () => { - handleClickDataPipes() - } - ) - } - - onMounted(() => { - scriptDetail(Number(route.params.dataPipeId)).then( - (res: ScriptDetail) => { - variables.name = res.name - variables.type = res.type - variables.content = res.content - } - ) - }) - - return { t, ...toRefs(variables), handleClickDataPipes, handleAdd } - }, - render() { - return ( - - - {{ - header: () => ( - - - - {this.t('data_pipes.data_pipes')} - - {this.name} - - - ), - 'header-extra': () => ( - - - {this.t('data_pipes.cancel')} - - - {this.t('data_pipes.save')} - - - ) - }} - - - - {this.t('data_pipes.name')} - - - - {{ - default: () => {this.t('data_pipes.name_tips')}, - trigger: () => ( - - - - ) - }} - - - - - - - - - ) - } -}) - -export default DataPipesEdit diff --git a/seatunnel-ui/src/views/data-pipes/list/components/delete-modal.tsx b/seatunnel-ui/src/views/data-pipes/list/components/delete-modal.tsx deleted file mode 100644 index 4fb4b862f..000000000 --- a/seatunnel-ui/src/views/data-pipes/list/components/delete-modal.tsx +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent } from 'vue' -import { useI18n } from 'vue-i18n' -import Modal from '@/components/modal' -import type { PropType } from 'vue' - -const props = { - showModal: { - type: Boolean as PropType, - default: false - }, - row: { - type: Object as PropType, - default: {} - } -} - -const DeleteModal = defineComponent({ - props, - emits: ['cancelModal', 'confirmModal'], - setup(props, ctx) { - const { t } = useI18n() - - const handleCancel = () => { - ctx.emit('cancelModal', props.showModal) - } - - const handleConfirm = () => { - ctx.emit('confirmModal') - } - - return { t, handleCancel, handleConfirm } - }, - render() { - return ( - - {{ - default: () => ( - {this.t('data_pipes.data_pipes_delete_tips')} - ) - }} - - ) - } -}) - -export default DeleteModal diff --git a/seatunnel-ui/src/views/data-pipes/list/components/publish-modal.tsx b/seatunnel-ui/src/views/data-pipes/list/components/publish-modal.tsx deleted file mode 100644 index b4eddeb6a..000000000 --- a/seatunnel-ui/src/views/data-pipes/list/components/publish-modal.tsx +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent } from 'vue' -import { useI18n } from 'vue-i18n' -import Modal from '@/components/modal' -import type { PropType } from 'vue' - -const props = { - showModal: { - type: Boolean as PropType, - default: false - }, - row: { - type: Object as PropType, - default: {} - } -} - -const PublishModal = defineComponent({ - props, - emits: ['cancelModal', 'confirmModal'], - setup(props, ctx) { - const { t } = useI18n() - - const handleCancel = () => { - ctx.emit('cancelModal', props.showModal) - } - - const handleConfirm = () => { - ctx.emit('confirmModal') - } - - return { t, handleCancel, handleConfirm } - }, - render() { - return ( - - {{ - default: () => ( - {this.t('data_pipes.data_pipes_publish_tips')} - ) - }} - - ) - } -}) - -export default PublishModal diff --git a/seatunnel-ui/src/views/data-pipes/list/index.tsx b/seatunnel-ui/src/views/data-pipes/list/index.tsx deleted file mode 100644 index 81e68fa73..000000000 --- a/seatunnel-ui/src/views/data-pipes/list/index.tsx +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent, onMounted, toRefs } from 'vue' -import { useI18n } from 'vue-i18n' -import { useTable } from './use-table' -import { NButton, NCard, NDataTable, NPagination, NSpace } from 'naive-ui' -import { useRouter } from 'vue-router' -import DeleteModal from './components/delete-modal' -import PublishModal from './components/publish-modal' -import type { Router } from 'vue-router' - -const DataPipesList = defineComponent({ - setup() { - const { t } = useI18n() - const router: Router = useRouter() - const { - state, - createColumns, - getTableData, - handleConfirmDeleteModal, - handleConfirmPublishModal - } = useTable() - - const requestData = () => { - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - } - - const handleCancelDeleteModal = () => { - state.showDeleteModal = false - } - - const handleCancelPublishModal = () => { - state.showPublishModal = false - } - - const handleCreate = () => { - router.push({ path: '/data-pipes/create' }) - } - - const handlePageSize = () => { - state.pageNo = 1 - requestData() - } - - onMounted(() => { - createColumns(state) - requestData() - }) - - return { - t, - ...toRefs(state), - handleCancelDeleteModal, - handleConfirmDeleteModal, - handleCancelPublishModal, - handleConfirmPublishModal, - handleCreate, - requestData, - handlePageSize - } - }, - render() { - return ( - - - {{ - 'header-extra': () => ( - - {this.t('data_pipes.create')} - - ) - }} - - - - - - - - - - - - - ) - } -}) - -export default DataPipesList diff --git a/seatunnel-ui/src/views/data-pipes/list/use-table.ts b/seatunnel-ui/src/views/data-pipes/list/use-table.ts deleted file mode 100644 index 69ab9e79b..000000000 --- a/seatunnel-ui/src/views/data-pipes/list/use-table.ts +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { reactive, ref, h } from 'vue' -import { useI18n } from 'vue-i18n' -import { NSpace, NButton, NTag } from 'naive-ui' -import { scriptList, scriptDelete, scriptPublish } from '@/service/script' -import { taskExecute } from '@/service/task' -import { useRouter } from 'vue-router' -import { getTableColumn } from '@/common/table' -import type { ResponseTable } from '@/service/types' -import type { ScriptDetail } from '@/service/script/types' -import type { Router } from 'vue-router' - -export function useTable() { - const { t } = useI18n() - const router: Router = useRouter() - const state = reactive({ - columns: [], - tableData: [], - pageNo: ref(1), - pageSize: ref(10), - totalPage: ref(1), - row: {}, - loading: ref(false), - showDeleteModal: ref(false), - showPublishModal: ref(false) - }) - - const createColumns = (state: any) => { - state.columns = [ - ...getTableColumn([{ key: 'id', title: t('data_pipes.id') }]), - { - title: t('data_pipes.name'), - key: 'name', - render: (row: ScriptDetail) => { - return h( - NButton, - { - text: true, - type: 'primary', - onClick: () => { - router.push({ - path: `/data-pipes/${row.id}` - }) - } - }, - { - default: () => row.name - } - ) - } - }, - { - title: t('data_pipes.state'), - key: 'status', - render: (row: ScriptDetail) => { - if (row.status === 'published') { - return h( - NTag, - { type: 'info' }, - { default: () => t('tasks.published') } - ) - } else { - return h( - NTag, - { type: 'default' }, - { default: () => t('tasks.unpublished') } - ) - } - } - }, - { - title: t('data_pipes.create_time'), - key: 'createTime' - }, - { - title: t('data_pipes.update_time'), - key: 'updateTime' - }, - { - title: t('data_pipes.operation'), - key: 'operation', - render: (row: ScriptDetail) => - h(NSpace, null, { - default: () => [ - h( - NButton, - { - text: true, - disabled: row.status !== 'published', - onClick: () => handleExecute(row) - }, - { - default: () => t('data_pipes.execute') - } - ), - h( - NButton, - { - text: true, - disabled: row.status === 'published', - onClick: () => { - router.push({ - path: `/data-pipes/${row.id}/edit` - }) - } - }, - { - default: () => t('data_pipes.edit') - } - ), - h( - NButton, - { - text: true, - disabled: row.status === 'published', - onClick: () => handlePublish(row) - }, - { default: () => t('data_pipes.publish') } - ), - h( - NButton, - { - text: true, - disabled: row.status === 'published', - onClick: () => handleDelete(row) - }, - { default: () => t('data_pipes.delete') } - ) - ] - }) - } - ] - } - - const handleDelete = (row: ScriptDetail) => { - state.showDeleteModal = true - state.row = row - } - - const handleConfirmDeleteModal = () => { - if (state.tableData.length === 1 && state.pageNo > 1) { - --state.pageNo - } - - scriptDelete((state.row as ScriptDetail).id as number).then(() => { - state.showDeleteModal = false - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - }) - } - - const handlePublish = (row: ScriptDetail) => { - state.showPublishModal = true - state.row = row - } - - const handleConfirmPublishModal = () => { - scriptPublish((state.row as ScriptDetail).id as number).then(() => { - state.showPublishModal = false - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - }) - } - - const handleExecute = (row: ScriptDetail) => { - taskExecute(row.id, { - objectType: 0, - executeType: 0 - }).then(() => { - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - }) - } - - const getTableData = (params: any) => { - if (state.loading) return - state.loading = true - scriptList(params).then((res: ResponseTable | []>) => { - state.tableData = res.data as any - state.totalPage = res.data.totalPage - state.loading = false - }) - } - - return { - state, - createColumns, - handleConfirmDeleteModal, - handleConfirmPublishModal, - getTableData - } -} diff --git a/seatunnel-ui/src/views/datasource/list/index.tsx b/seatunnel-ui/src/views/datasource/list/index.tsx index 052757494..260cfdfac 100644 --- a/seatunnel-ui/src/views/datasource/list/index.tsx +++ b/seatunnel-ui/src/views/datasource/list/index.tsx @@ -52,7 +52,7 @@ const DatasourceList = defineComponent({ const { getColumns } = useColumns((id: string, type: 'edit' | 'delete') => { if (type === 'edit') { router.push({ name: 'datasource-edit', params: { id } }) - } else { + } else if(type === 'delete'){ deleteRecord(id) } }) diff --git a/seatunnel-ui/src/views/datasource/list/use-columns.ts b/seatunnel-ui/src/views/datasource/list/use-columns.ts index c67d0adec..81b31a76d 100644 --- a/seatunnel-ui/src/views/datasource/list/use-columns.ts +++ b/seatunnel-ui/src/views/datasource/list/use-columns.ts @@ -23,7 +23,6 @@ import { getTableColumn } from '@/common/table' import { useTableOperation } from '@/hooks' import { EditOutlined } from '@vicons/antd' -import ResourceAuth from '@/components/resource-auth' export function useColumns(onCallback: Function) { const { t } = useI18n() @@ -81,33 +80,25 @@ export function useColumns(onCallback: Function) { title: t('datasource.update_time'), key: 'updateTime', }, - { - title: t('data_pipes.operation'), + useTableOperation({ + title: t('datasource.operation'), key: 'operation', - render: (row: any) => - h(NSpace, null, { - default: () => [ - h( - NButton, + buttons: [ { - text: true, - onClick: () => void onCallback(row.id, 'edit') + text: t('datasource.edit'), + icon: h(EditOutlined), + onClick: (rowData) => void onCallback(rowData.id, 'edit') }, { - default: () => t('datasource.edit') - } - ), - h( - NButton, - { - text: true, - onClick: () => void onCallback(row.id, 'delete') - }, - { default: () => t('datasource.delete') } - ) + isDelete: true, + text: t('datasource.delete'), + onPositiveClick: (rowData) => void onCallback(rowData.id, 'delete'), + negativeText: t('datasource.cancel'), + positiveText: t('datasource.confirm'), + popTips: t('datasource.delete_confirm') + } ] }) - } ] } diff --git a/seatunnel-ui/src/views/jobs/list/index.tsx b/seatunnel-ui/src/views/jobs/list/index.tsx deleted file mode 100644 index e9879527c..000000000 --- a/seatunnel-ui/src/views/jobs/list/index.tsx +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { defineComponent, onMounted, toRefs } from 'vue' -import { - NSpace, - NCard, - NButton, - NInput, - NDataTable, - NPagination -} from 'naive-ui' -import { useI18n } from 'vue-i18n' -import { useTable } from './use-table' - -const JobsList = defineComponent({ - setup() { - const { t } = useI18n() - const { state, createColumns, getTableData } = useTable() - - const handleSearch = () => { - state.pageNo = 1 - requestData() - } - - const handlePageSize = () => { - state.pageNo = 1 - requestData() - } - - const requestData = () => { - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo, - name: state.name - }) - } - - onMounted(() => { - createColumns(state) - requestData() - }) - - return { t, handleSearch, ...toRefs(state), requestData, handlePageSize } - }, - render() { - return ( - - - {{ - 'header-extra': () => ( - - - - {this.t('jobs.search')} - - - ) - }} - - - - - - - - - - - ) - } -}) - -export default JobsList diff --git a/seatunnel-ui/src/views/jobs/list/use-table.ts b/seatunnel-ui/src/views/jobs/list/use-table.ts deleted file mode 100644 index 488abbfee..000000000 --- a/seatunnel-ui/src/views/jobs/list/use-table.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { useI18n } from 'vue-i18n' -import { h, reactive, ref } from 'vue' -import { NButton, NSpace } from 'naive-ui' -import { taskJobList, taskExecute, taskRecycle } from '@/service/task' -import { getTableColumn } from '@/common/table' -import type { ResponseTable } from '@/service/types' -import type { JobDetail } from '@/service/task/types' - -export function useTable() { - const { t } = useI18n() - - const state = reactive({ - columns: [], - tableData: [], - name: ref(null), - pageNo: ref(1), - pageSize: ref(10), - totalPage: ref(1), - loading: ref(false) - }) - - const createColumns = (state: any) => { - state.columns = [ - ...getTableColumn([ - { key: 'scriptId', title: t('jobs.script_id') }, - { key: 'jobId', title: t('jobs.job_id') } - ]), - { - title: t('jobs.data_pipe_name'), - key: 'datapipeName' - }, - { - title: t('jobs.plan'), - key: 'jobPlan' - }, - { - title: t('jobs.create_time'), - key: 'createTime' - }, - { - title: t('jobs.operation'), - key: 'operation', - render: (row: JobDetail) => - h(NSpace, null, { - default: () => [ - h( - NButton, - { - text: true, - disabled: row.jobStatus === 'OFFLINE', - onClick: () => handleExecute(row.jobId) - }, - { default: () => t('jobs.execute') } - ), - h( - NButton, - { - text: true, - disabled: row.jobStatus === 'OFFLINE', - onClick: () => handleRecycle(row.jobId) - }, - { default: () => t('jobs.recycle') } - ) - ] - }) - } - ] - } - - const getTableData = (params: any) => { - if (state.loading) return - state.loading = true - taskJobList({ ...params }).then( - (res: ResponseTable | []>) => { - state.tableData = res.data as any - state.totalPage = res.data.totalPage - state.loading = false - } - ) - } - - const handleExecute = (id: number) => { - taskExecute(id, { - objectType: 1, - executeType: 0 - }).then(() => { - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - }) - } - - const handleRecycle = (id: number) => { - taskRecycle(id).then(() => { - getTableData({ - pageSize: state.pageSize, - pageNo: state.pageNo - }) - }) - } - - return { state, createColumns, getTableData } -} diff --git a/seatunnel-ui/src/views/login/use-form.ts b/seatunnel-ui/src/views/login/use-form.ts index 79e23056c..c9fee48d0 100644 --- a/seatunnel-ui/src/views/login/use-form.ts +++ b/seatunnel-ui/src/views/login/use-form.ts @@ -57,7 +57,8 @@ export function useForm() { const handleLogin = () => { userLogin({ ...state.loginForm }).then((res: any) => { userStore.setUserInfo(res) - router.push({ name: 'tasks' }) + console.log('login-jumps') + router.push({ path: '/tasks' }) }) } diff --git a/seatunnel-ui/src/views/task/synchronization-definition/dag/canvas/dag-shape.ts b/seatunnel-ui/src/views/task/synchronization-definition/dag/canvas/dag-shape.ts index 6cb5ad958..18ebb9cfd 100644 --- a/seatunnel-ui/src/views/task/synchronization-definition/dag/canvas/dag-shape.ts +++ b/seatunnel-ui/src/views/task/synchronization-definition/dag/canvas/dag-shape.ts @@ -93,7 +93,7 @@ export function formatLayout( cols?: number, rows?: number ) { - let layoutFunc = null + let layoutFunc : any = null const layoutConfig: any = { nodesep: 50, padding: 50, diff --git a/seatunnel-ui/src/views/task/synchronization-definition/index.tsx b/seatunnel-ui/src/views/task/synchronization-definition/index.tsx index ce8c7bffe..671cb73b1 100644 --- a/seatunnel-ui/src/views/task/synchronization-definition/index.tsx +++ b/seatunnel-ui/src/views/task/synchronization-definition/index.tsx @@ -39,7 +39,7 @@ const SynchronizationDefinition = defineComponent({ const route = useRoute() const router = useRouter() const { variables, createColumns, getTableData } = useTable() - + console.log('SynchronizationDefinition') const requestData = () => { getTableData({ pageSize: variables.pageSize, diff --git a/seatunnel-ui/src/views/task/synchronization-instance/detail/task-definition.tsx b/seatunnel-ui/src/views/task/synchronization-instance/detail/task-definition.tsx index 34552b5b6..7a344174a 100644 --- a/seatunnel-ui/src/views/task/synchronization-instance/detail/task-definition.tsx +++ b/seatunnel-ui/src/views/task/synchronization-instance/detail/task-definition.tsx @@ -68,7 +68,7 @@ const TaskDefinition = defineComponent({ const formatData = () => { const obj = jobConfig.value.env - const arr = [] + const arr : any = [] for (const i in obj) { arr.push({ label: String(i), value: obj[i] }) } diff --git a/seatunnel-ui/src/views/virtual-tables/list/use-columns.ts b/seatunnel-ui/src/views/virtual-tables/list/use-columns.ts index 860dd0000..86b881f0b 100644 --- a/seatunnel-ui/src/views/virtual-tables/list/use-columns.ts +++ b/seatunnel-ui/src/views/virtual-tables/list/use-columns.ts @@ -19,6 +19,7 @@ import { h, ref, watch, onMounted } from 'vue' import { useI18n } from 'vue-i18n' import { EditOutlined } from '@vicons/antd' import { NButton, NSpace } from 'naive-ui' +import {useTableOperation} from "@/hooks"; //import type { TableColumns, VirtualTableRecord } from '../types' export function useColumns(onCallback: Function) { @@ -70,33 +71,25 @@ export function useColumns(onCallback: Function) { //render: (rowData: VirtualTableRecord) => // renderTableTime(rowData.createTime) }, - { + useTableOperation({ title: t('virtual_tables.operation'), key: 'operation', - render: (row: any) => - h(NSpace, null, { - default: () => [ - h( - NButton, - { - text: true, - onClick: () => void onCallback(row.tableId, 'edit') - }, - { - default: () => t('virtual_tables.edit') - } - ), - h( - NButton, - { - text: true, - onClick: () => void onCallback(row.tableId, 'delete') - }, - { default: () => t('virtual_tables.delete') } - ) - ] - }) - } + buttons: [ + { + text: t('datasource.edit'), + icon: h(EditOutlined), + onClick: (rowData) => void onCallback(rowData.tableId, 'edit') + }, + { + isDelete: true, + text: t('datasource.delete'), + onPositiveClick: (rowData) => void onCallback(rowData.tableId, 'delete'), + negativeText: t('datasource.cancel'), + positiveText: t('datasource.confirm'), + popTips: t('datasource.delete_confirm') + } + ] + }) ] return columns } diff --git a/seatunnel-ui/tsconfig.json b/seatunnel-ui/tsconfig.json index 8806a48df..815e2c0d7 100644 --- a/seatunnel-ui/tsconfig.json +++ b/seatunnel-ui/tsconfig.json @@ -4,6 +4,7 @@ "module": "esnext", "moduleResolution": "node", "strict": true, + "noImplicitAny": false, "jsx": "preserve", "sourceMap": true, "resolveJsonModule": true, diff --git a/seatunnel-ui/vite.config.ts b/seatunnel-ui/vite.config.ts index d7498d01d..8a3225a0a 100644 --- a/seatunnel-ui/vite.config.ts +++ b/seatunnel-ui/vite.config.ts @@ -38,7 +38,8 @@ export default defineConfig({ resolve: { alias: { '@': path.resolve(__dirname, 'src'), - 'vue-i18n': 'vue-i18n/dist/vue-i18n.cjs.js' + 'vue-i18n': 'vue-i18n/dist/vue-i18n.cjs.js', + '@intlify/shared': '@intlify/shared/dist/shared.cjs.js' } }, server: { diff --git a/seatunnel-web-dist/pom.xml b/seatunnel-web-dist/pom.xml index 7f15a9f09..0623bd2f6 100644 --- a/seatunnel-web-dist/pom.xml +++ b/seatunnel-web-dist/pom.xml @@ -32,62 +32,17 @@ true - 21.5.0.0 - 0.3.2-patch11 42.4.3 - 2.1.0.10 9.2.1.jre8 8.0.28 - - - - com.github.eirslett - frontend-maven-plugin - 1.11.3 - - ${project.basedir}/../${frontend.project.name} - - - - install node and npm - - install-node-and-npm - - - v16.16.0 - 8.19.2 - - - - install - - npm - - generate-resources - - install --ignore-scripts - - - - - - - - - - - - - - - release + false release true @@ -99,24 +54,61 @@ org.apache.seatunnel seatunnel-app ${project.version} - provided org.apache.seatunnel seatunnel-datasource-client ${project.version} - provided apache-seatunnel-web-${project.version} + + com.github.eirslett + frontend-maven-plugin + 1.11.3 + + ${project.basedir}/../seatunnel-ui + + + + install node and npm + + install-node-and-npm + + + v16.16.0 + 8.19.2 + + + + install + + npm + + generate-resources + + install --ignore-scripts + + + + build + + npm + + + run build:prod + + + + maven-assembly-plugin - seatunnel-web + bin single @@ -125,7 +117,20 @@ src/main/assembly/seatunnel-web.xml - false + true + + + + src + + single + + package + + + src/main/assembly/assembly-src.xml + + true @@ -188,7 +193,7 @@ org.apache.seatunnel - datasource-jdbc-mysql + datasource-hive ${project.version} provided @@ -200,7 +205,7 @@ org.apache.seatunnel - datasource-jdbc-oracle + datasource-jdbc-mysql ${project.version} provided @@ -212,7 +217,7 @@ org.apache.seatunnel - datasource-jdbc-postgresql + datasource-jdbc-oracle ${project.version} provided @@ -224,7 +229,7 @@ org.apache.seatunnel - datasource-jdbc-tidb + datasource-jdbc-postgresql ${project.version} provided @@ -236,7 +241,7 @@ org.apache.seatunnel - datasource-jdbc-redshift + datasource-jdbc-tidb ${project.version} provided @@ -298,7 +303,7 @@ org.apache.seatunnel - datasource-s3redshift + datasource-starrocks ${project.version} provided @@ -310,7 +315,7 @@ org.apache.seatunnel - datasource-starrocks + datasource-mysql-cdc ${project.version} provided @@ -322,7 +327,7 @@ org.apache.seatunnel - datasource-mysql-cdc + datasource-s3 ${project.version} provided @@ -334,7 +339,7 @@ org.apache.seatunnel - datasource-s3 + datasource-sqlserver-cdc ${project.version} provided @@ -344,9 +349,10 @@ + org.apache.seatunnel - datasource-sqlserver-cdc + datasource-mongodb ${project.version} provided @@ -357,24 +363,27 @@ - - - com.oracle.database.jdbc - ojdbc8 - ${oracle-jdbc.version} + org.apache.seatunnel + datasource-jdbc-db2 + ${project.version} provided + + + * + * + + - + - com.clickhouse - clickhouse-jdbc - ${clickhouse.version} + com.oracle.database.jdbc + ojdbc8 + ${oracle-jdbc.version} provided - org.postgresql postgresql @@ -382,14 +391,6 @@ provided - - com.amazon.redshift - redshift-jdbc42 - ${redshift.version} - provided - - - com.microsoft.sqlserver mssql-jdbc @@ -397,7 +398,6 @@ provided - mysql mysql-connector-java @@ -405,11 +405,52 @@ provided + + apache-seatunnel-web-${project.version} + + com.github.eirslett + frontend-maven-plugin + 1.11.3 + + ${project.basedir}/../seatunnel-ui + + + + install node and npm + + install-node-and-npm + + + v16.16.0 + 8.19.2 + + + + install + + npm + + generate-resources + + install --ignore-scripts + + + + build + + npm + + + run build:prod + + + + maven-assembly-plugin diff --git a/seatunnel-web-dist/release-docs/LICENSE b/seatunnel-web-dist/release-docs/LICENSE index b01312a9a..8be15e811 100644 --- a/seatunnel-web-dist/release-docs/LICENSE +++ b/seatunnel-web-dist/release-docs/LICENSE @@ -201,9 +201,9 @@ limitations under the License. ======================================================================= -Apache SeaTunnel (Incubating) Subcomponents: +Apache SeaTunnel Subcomponents: -The Apache SeaTunnel(Incubating) project contains subcomponents with separate copyright +The Apache SeaTunnel project contains subcomponents with separate copyright notices and license terms. Your use of the source code for the these subcomponents is subject to the terms and conditions of the following licenses. @@ -221,19 +221,32 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) ClassMate (com.fasterxml:classmate:1.3.1 - http://github.com/cowtowncoder/java-classmate) (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.13 - https://commons.apache.org/proper/commons-codec/) (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/) - (Apache License, Version 2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.12.0 - http://commons.apache.org/proper/commons-lang/) + (Apache License, Version 2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.4 - http://commons.apache.org/proper/commons-lang/) (Apache License, Version 2.0) Apache Commons IO (commons-io:commons-io:2.11.0 - https://mvnrepository.com/artifact/commons-io/commons-io/2.11.0) + (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.20 - https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.20) (Apache License, Version 2.0) config (com.typesafe:config:1.3.3 - https://github.com/lightbend/config) - (Apache License, Version 2.0) SeaTunnel Common (org.apache.seatunnel:seatunnel-common:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-common) - (Apache License, Version 2.0) SeaTunnel API (org.apache.seatunnel:seatunnel-api:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-api) - (Apache License, Version 2.0) SeaTunnel Plugin Discovery (org.apache.seatunnel:seatunnel-plugin-discovery:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-plugin-discovery) - (Apache License, Version 2.0) SeaTunnel Engine Client (org.apache.seatunnel:seatunnel-engine-client:2.3.1 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-engine-client) - - (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:19.0 - https://github.com/google/guava/guava) + (The Apache Software License, Version 2.0) hazelcast (com.hazelcast:hazelcast:5.1 - https://github.com/hazelcast/hazelcast) + (Apache License, Version 2.0) SeaTunnel Common (org.apache.seatunnel:seatunnel-common:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-common) + (Apache License, Version 2.0) SeaTunnel API (org.apache.seatunnel:seatunnel-api:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-api) + (Apache License, Version 2.0) SeaTunnel Config Base (org.apache.seatunnel:seatunnel-config-base:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-config-base) + (Apache License, Version 2.0) SeaTunnel Config Shade (org.apache.seatunnel:seatunnel-config-shade:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-config-shade) + (Apache License, Version 2.0) SeaTunnel Core Starter (org.apache.seatunnel:seatunnel-core-starter:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-core-starter) + (Apache License, Version 2.0) SeaTunnel Core Starter (org.apache.seatunnel:seatunnel-core-starter:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-core-starter) + (Apache License, Version 2.0) SeaTunnel Engine Client (org.apache.seatunnel:seatunnel-engine-client:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-engine-client) + (Apache License, Version 2.0) SeaTunnel Engine Common (org.apache.seatunnel:seatunnel-engine-common:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-engine-common) + (Apache License, Version 2.0) SeaTunnel Engine Core (org.apache.seatunnel:seatunnel-engine-core:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-engine-core) + (Apache License, Version 2.0) SeaTunnel Guava (org.apache.seatunnel:seatunnel-guava:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-guava) + (Apache License, Version 2.0) SeaTunnel Jackson (org.apache.seatunnel:seatunnel-jackson:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-jackson) + (Apache License, Version 2.0) SeaTunnel Plugin Discovery (org.apache.seatunnel:seatunnel-plugin-discovery:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-plugin-discovery) + (Apache License, Version 2.0) SeaTunnel Transform V2 (org.apache.seatunnel:seatunnel-transforms-v2:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-transforms-v2) + (Apache License, Version 2.0) SeaTunnel Serializer Api (org.apache.seatunnel:serializer-api:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/serializer-api) + (Apache License, Version 2.0) SeaTunnel Serializer Protobuf (org.apache.seatunnel:serializer-protobuf:2.3.3 - https://mvnrepository.com/artifact/org.apache.seatunnel/serializer-protobuf) + + (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:19.0 - https://github.com/google/guava) (Apache License 2.0) Hibernate Validator Engine (org.hibernate.validator:hibernate-validator:6.2.2.Final - http://hibernate.org/validator/hibernate-validator) - (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.12.7 - http://github.com/FasterXML/jackson) - (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.12.7 - https://github.com/FasterXML/jackson-core) - (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.12.7.1 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.12.6 - https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.12.6) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.12.6 - https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/2.12.6) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.12.6 - https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.12.6) (The Apache Software License, Version 2.0) Jackson datatype: jdk8 (com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8) (The Apache Software License, Version 2.0) Jackson datatype: JSR310 (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310) (The Apache Software License, Version 2.0) Defect Detection Metadata: JSR305 (com.google.code.findbugs:jsr305:3.0.0 - http://findbugs.sourceforge.net/) @@ -254,7 +267,9 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) JJWT :: API (io.jsonwebtoken:jjwt-api:0.10.7 - https://github.com/jwtk/jjwt/jjwt-api) (Apache License, Version 2.0) JJWT :: Impl (io.jsonwebtoken:jjwt-impl:0.10.7 - https://github.com/jwtk/jjwt/jjwt-impl) (Apache License, Version 2.0) JJWT :: Extensions :: Jackson (io.jsonwebtoken:jjwt-jackson:0.10.7 - https://github.com/jwtk/jjwt/jjwt-jackson) - (The Apache Software License, Version 2.0) Apache Log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/) + (The Apache Software License, Version 2.0) Apache Log4j API (org.apache.logging.log4j:log4j-api:2.17.1 - https://logging.apache.org/log4j/2.x/) + (The Apache Software License, Version 2.0) Apache Log4j to SLF4J Adapter (org.apache.logging.log4j:log4j-to-slf4j:2.17.1 - https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-to-slf4j/2.17.1) + (The Apache Software License, Version 2.0) Log4j Implemented Over SLF4J (org.slf4j:log4j-over-slf4j:1.7.25 - https://mvnrepository.com/artifact/org.slf4j/log4j-over-slf4j/1.7.25) (The Apache Software License, Version 2.0) MapStruct Core (org.mapstruct:mapstruct:1.0.0.Final - http://mapstruct.org/mapstruct/) (The Apache Software License, Version 2.0) mybatis (org.mybatis:mybatis:3.5.10 - http://www.mybatis.org/mybatis-3) (The Apache Software License, Version 2.0) mybatis-spring (org.mybatis:mybatis-spring:2.0.7 - http://www.mybatis.org/spring/) @@ -274,7 +289,6 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) spring-boot-starter-jdbc (org.springframework.boot:spring-boot-starter-jdbc:2.6.8 - https://spring.io/projects/spring-boot) (Apache License, Version 2.0) spring-boot-starter-jetty (org.springframework.boot:spring-boot-starter-jetty:2.6.8 - https://spring.io/projects/spring-boot) (Apache License, Version 2.0) spring-boot-starter-json (org.springframework.boot:spring-boot-starter-json:2.6.8 - https://spring.io/projects/spring-boot) - (Apache License, Version 2.0) spring-boot-starter-logging (org.springframework.boot:spring-boot-starter-logging:2.6.8 - https://spring.io/projects/spring-boot) (Apache License, Version 2.0) spring-boot-starter-web (org.springframework.boot:spring-boot-starter-web:2.6.8 - https://spring.io/projects/spring-boot) (Apache License, Version 2.0) Spring Context (org.springframework:spring-context:5.3.20 - https://github.com/spring-projects/spring-framework) (Apache License, Version 2.0) Spring Core (org.springframework:spring-core:5.3.20 - https://github.com/spring-projects/spring-framework) @@ -295,11 +309,16 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) springfox-swagger-ui (io.springfox:springfox-swagger-ui:2.6.1 - https://github.com/springfox/springfox) (The Apache Software License, Version 2.0) springfox-swagger2 (io.springfox:springfox-swagger2:2.6.1 - https://github.com/springfox/springfox) (Apache License 2.0) swagger-annotations (io.swagger:swagger-annotations:1.5.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Apache License 2.0) swagger-annotations (io.swagger:swagger-annotations:2.1.14 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) (Apache License 2.0) swagger-models (io.swagger:swagger-models:1.5.10 - https://github.com/swagger-api/swagger-core/modules/swagger-models) (Apache License, Version 2.0) tomcat-embed-el (org.apache.tomcat.embed:tomcat-embed-el:9.0.63 - https://tomcat.apache.org/) - (Apache License, Version 2.0) scala-liberary (org.scala-lang:scala-library:jar:2.11.12 - https://www.scala-lang.org/) + (Apache License, Version 2.0) scala-liberary (org.scala-lang:scala-library:jar:2.11.12 - https://www.scala-lang.org/) (Apache License, Version 2.0) gson (com.google.code.gson:gson:jar:2.8.6 - https://mvnrepository.com/artifact/com.google.code.gson/gson/2.8.6) - (Apache License, Version 2.0) cron-utils (com.cronutils:cron-utils:jar:9.1.6 - https://mvnrepository.com/artifact/com.cronutils/cron-utils/9.1.6) + (Apache License, Version 2.0) jcommander (com.beust:jcommander:1.81 - https://jcommander.org) + (The Apache Software License, Version 2.0) protostuff (io.protostuff:protostuff-collectionschema:1.8.0 - https://github.com/protostuff/protostuff) + (The Apache Software License, Version 2.0) protostuff (io.protostuff:protostuff-core:1.8.0 - https://github.com/protostuff/protostuff) + (The Apache Software License, Version 2.0) protostuff (io.protostuff:protostuff-api:1.8.0 - https://github.com/protostuff/protostuff) + (The Apache Software License, Version 2.0) protostuff (io.protostuff:protostuff-runtime:1.8.0 - https://github.com/protostuff/protostuff) ======================================================================== @@ -313,41 +332,56 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (EPL 2.0) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:1.3.5 - https://projects.eclipse.org/projects/ee4j.ca) (EPL 2.0) Jakarta Servlet (jakarta.servlet:jakarta.servlet-api:4.0.4 - https://projects.eclipse.org/projects/ee4j.servlet) (EPL 2.0) Jakarta WebSocket - Server API (jakarta.websocket:jakarta.websocket-api:1.1.2 - https://projects.eclipse.org/projects/ee4j.websocket) - (EPL 1.0) Logback Classic Module (ch.qos.logback:logback-classic:1.2.11 - http://logback.qos.ch/logback-classic) - (EPL 1.0) Logback Core Module (ch.qos.logback:logback-core:1.2.11 - http://logback.qos.ch/logback-core) + (EPL 1.0) Logback Classic Module (ch.qos.logback:logback-classic:1.2.3 - http://logback.qos.ch/logback-classic) + (EPL 1.0) Logback Core Module (ch.qos.logback:logback-core:1.2.3 - http://logback.qos.ch/logback-core) (EPL 1.0) h2 (com.h2database:h2:2.1.214 - https://h2database.com/) ======================================================================== -CDDL License +MIT License ======================================================================== -The following components are provided under the CDDL License. See project link for details. +The following components are provided under the MIT License. See project link for details. The text of each license is also included at licenses/LICENSE-[project].txt. - - (CDDL) javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250) - (CDDL and GPL and GPL 2.0) javax.el (org.glassfish:javax.el:3.0.0 https://mvnrepository.com/artifact/org.glassfish/javax.el/3.0.0) - + (MIT License) JCL 1.2 Implemented Over SLF4J (org.slf4j:jcl-over-slf4j:1.7.25 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.25 - http://www.slf4j.org) + ======================================================================== -MIT License +UI related licenses ======================================================================== -The following components are provided under the MIT License. See project link for details. -The text of each license is also included at licenses/LICENSE-[project].txt. +The following components are used in UI.See project link for details. +The text of each license is also included at licenses/ui-licenses/LICENSE-[project].txt. + +====================================================================== +MIT licenses +====================================================================== + (MIT) @antv/layout 0.1.31: https://github.com/antvis/layout + (MIT) @antv/x6 1.30.1: https://github.com/antvis/x6 + (MIT) @antv/x6-vue-shape 1.5.3: https://github.com/antvis/x6 + (MIT) @vueuse/core 9.13.0: https://github.com/vueuse/vueuse + (MIT) autoprefixer 10.4.13: https://www.npmjs.com/package/autoprefixer/v/10.4.13 + (MIT) axios 1.3.4: https://www.npmjs.com/package/axios/v/1.3.4 + (MIT) date-fns 2.29.3: https://www.npmjs.com/package/date-fns/v/2.29.3 + (MIT) date-fns-tz 2.0.0: https://www.npmjs.com/package/date-fns-tz/v/2.0.0 + (MIT) lodash 4.17.21: https://www.npmjs.com/package/lodash/v/4.17.21 + (MIT) monaco-editor 0.36.1: https://www.npmjs.com/package/monaco-editor/v/0.36.1 + (MIT) naive-ui 2.34.3: https://www.npmjs.com/package/naive-ui/v/2.34.3 + (MIT) nprogress 0.2.0: https://www.npmjs.com/package/nprogress/v/0.2.0 + (MIT) pinia 2.0.32: https://www.npmjs.com/package/pinia/v/2.0.32 + (MIT) pinia-plugin-persistedstate 3.1.0: https://www.npmjs.com/package/pinia-plugin-persistedstate/v/3.1.0 + (MIT) postcss 8.4.21: https://www.npmjs.com/package/postcss/v/8.4.21 + (MIT) screenfull 6.0.1: https://www.npmjs.com/package/screenfull/v/6.0.1 + (MIT) tailwindcss 3.2.7: https://www.npmjs.com/package/tailwindcss/v/3.2.7 + (MIT) vue 3.2.47: https://www.npmjs.com/package/vue/v/3.2.47 + (MIT) vue-i18n 9.2.2: https://www.npmjs.com/package/vue-i18n/v/9.2.2 + (MIT) vue-router 4.1.6: https://www.npmjs.com/package/vue-router/v/4.1.6 - (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.25 - http://www.slf4j.org) - (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.25 - http://www.slf4j.org) - (The MIT License) jsoup Java HTML Parser (org.jsoup:jsoup:1.14.3 - https://jsoup.org/) - (MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.36 - http://www.slf4j.org) ======================================================================== -MPL 1.1 licenses +Apache 2.0 License ======================================================================== -The following components are provided under a MPL 1.1 license. See project link for details. -The text of each license is also included at licenses/LICENSE-[project].txt. - (MPL-1.1) javassist (org.javassist:javassist:3.27.0-GA - https://github.com/jboss-javassist/javassist) + (Apache 2.0 License) echarts 5.4.1: https://www.npmjs.com/package/echarts/v/5.4.1 - - \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/NOTICE b/seatunnel-web-dist/release-docs/NOTICE index d4d6862ed..2efb0baa5 100644 --- a/seatunnel-web-dist/release-docs/NOTICE +++ b/seatunnel-web-dist/release-docs/NOTICE @@ -4,6 +4,21 @@ Copyright 2021-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +=============================================================================== + +ClassMate NOTICE + +=============================================================================== +Java ClassMate library was originally written by Tatu Saloranta (tatu.saloranta@iki.fi) + +Other developers who have contributed code are: + +* Brian Langel + +## Copyright + +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) + =============================================================================== @@ -33,7 +48,6 @@ Copyright (c) 2008 Alexander Beider & Stephen P. Morse. Apache Commons Collections NOTICE =============================================================================== - Apache Commons Collections Copyright 2001-2019 The Apache Software Foundation @@ -42,14 +56,89 @@ The Apache Software Foundation (http://www.apache.org/). =============================================================================== +Apache Commons Compress NOTICE + +=============================================================================== +Apache Commons Compress +Copyright 2002-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +--- + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + +--- + +The test file lbzip2_32767.bz2 has been copied from libbzip2's source +repository: + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org + +=============================================================================== + +Apache Commons IO NOTICE + +=============================================================================== +Apache Commons IO +Copyright 2002-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +=============================================================================== + Apache Commons Lang NOTICE =============================================================================== Apache Commons Lang -Copyright 2001-2021 The Apache Software Foundation +Copyright 2001-2015 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) =============================================================================== @@ -124,46 +213,43 @@ Test dependencies: * [SnakeYAML](https://bitbucket.org/asomov/snakeyaml/src) - Apache License 2.0 =============================================================================== -Notices for Eclipse Jetty - -========================= -This content is produced and maintained by the Eclipse Jetty project. + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +=============================================================================== -Project home: https://www.eclipse.org/jetty/ +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -Trademarks ----------- -Eclipse Jetty, and Jetty are trademarks of the Eclipse Foundation. +Jetty is dual licensed under both -Copyright ---------- -All contributions are the property of the respective authors or of -entities to which copyright has been assigned by the authors (eg. employer). + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html -Declared Project Licenses -------------------------- -This artifacts of this project are made available under the terms of: + and - * the Eclipse Public License v. 1.0 + * The Eclipse Public 1.0 License http://www.eclipse.org/legal/epl-v10.html - SPDX-License-Identifier: EPL-1.0 - or +Jetty may be distributed under either license. - * the Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0. - SPDX-License-Identifier: Apache-2.0 +------ +Eclipse -The following dependencies are EPL. +The following artifacts are EPL. * org.eclipse.jetty.orbit:org.eclipse.jdt.core -The following dependencies are EPL and ASL2. +The following artifacts are EPL and ASL2. * org.eclipse.jetty.orbit:javax.security.auth.message -The following dependencies are EPL and CDDL 1.0. + +The following artifacts are EPL and CDDL 1.0. * org.eclipse.jetty.orbit:javax.mail.glassfish -The following dependencies are CDDL + GPLv2 with classpath exception. + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html * javax.servlet:javax.servlet-api @@ -171,55 +257,72 @@ https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html * javax.transaction:javax.transaction-api * javax.websocket:javax.websocket-api +------ +Oracle OpenJDK + If ALPN is used to negotiate HTTP/2 connections, then the following -distribution may be included in the distribution or downloaded when ALPN -module is selected. These artifacts replace/modify OpenJDK classes. -The modifications are hosted at github and both modified and original -are under GPL v2 with classpath exceptions. -http://openjdk.java.net/legal/gplv2+ce.html +artifacts may be included in the distribution or downloaded when ALPN +module is selected. * java.sun.security.ssl -The following dependencies are licensed by the OW2 Foundation according to the +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the terms of http://asm.ow2.org/license.html - * org.ow2.asm:asm-commons - * org.ow2.asm:asm +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache -The following dependencies are ASL2 licensed. +The following artifacts are ASL2 licensed. - * org.apache.taglibs:taglibs-standard-spec - * org.apache.taglibs:taglibs-standard-impl +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -The following dependencies are ASL2 licensed. Based on selected classes from + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from following Apache Tomcat jars, all ASL2 licensed. - * org.mortbay.jasper:apache-jsp - * org.apache.tomcat:tomcat-jasper - * org.apache.tomcat:tomcat-juli - * org.apache.tomcat:tomcat-jsp-api - * org.apache.tomcat:tomcat-el-api - * org.apache.tomcat:tomcat-jasper-el - * org.apache.tomcat:tomcat-api - * org.apache.tomcat:tomcat-util-scan - * org.apache.tomcat:tomcat-util - * org.mortbay.jasper:apache-el - * org.apache.tomcat:tomcat-jasper-el - * org.apache.tomcat:tomcat-el-api +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay The following artifacts are CDDL + GPLv2 with classpath exception. + https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - * org.eclipse.jetty.toolchain:jetty-schemas +org.eclipse.jetty.toolchain:jetty-schemas -Cryptography ------------- -Content may contain encryption software. The country in which you are currently -may have restrictions on the import, possession, and use, and/or re-export to -another country, of encryption software. BEFORE using any encryption software, -please check the country's laws, regulations and policies concerning the import, -possession, or use, and re-export of encryption software, to see if this is -permitted. +------ +Assorted The UnixCrypt.java code implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, @@ -398,6 +501,46 @@ Spring Framework WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +=============================================================================== + +Spring Boot 2.6.8 + +=============================================================================== +Spring Boot 2.6.8 +Copyright (c) 2012-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +=============================================================================== + +Spring Framework 5.3.20 + +=============================================================================== +Spring Framework 5.3.20 +Copyright (c) 2002-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +=============================================================================== + +Swagger Core - ${pom.name} + +=============================================================================== +Swagger Core - ${pom.name} +Copyright (c) 2015. SmartBear Software Inc. +Swagger Core - ${pom.name} is licensed under Apache 2.0 license. +Copy of the Apache 2.0 license can be found in `LICENSE` file. + =============================================================================== Apache Tomcat NOTICE @@ -456,3 +599,55 @@ This software includes projects with other licenses -- see `doc/LICENSE.md`. =============================================================================== +hazelcast NOTICE + +========================================================================= +The packages: + +com.hazelcast.internal.util.collection +com.hazelcast.internal.util.concurrent + +and the classes: + +com.hazelcast.internal.util.QuickMath +com.hazelcast.client.impl.protocol.util.UnsafeBuffer +com.hazelcast.client.impl.protocol.util.BufferBuilder + +contain code originating from the Agrona project +(https://github.com/real-logic/Agrona). + +The class com.hazelcast.internal.util.HashUtil contains code originating +from the Koloboke project (https://github.com/OpenHFT/Koloboke). + +The class classloading.ThreadLocalLeakTestUtils contains code originating +from the Tomcat project (https://github.com/apache/tomcat). + +com.hazelcast.internal.cluster.fd.PhiAccrualFailureDetector contains code originating +from the Akka project (https://github.com/akka/akka/). + +The package com.hazelcast.internal.json contains code originating +from minimal-json project (https://github.com/ralfstx/minimal-json). + +The class com.hazelcast.instance.impl.MobyNames contains code originating +from The Moby Project (https://github.com/moby/moby). + +The class com.hazelcast.internal.util.graph.BronKerboschCliqueFinder contains code +originating from The JGraphT Project (https://github.com/jgrapht/jgrapht). + +The packages: +com.hazelcast.sql +com.hazelcast.jet.sql + +contain code originating from the Apache Calcite (https://github.com/apache/calcite) + +The class com.hazelcast.jet.kafka.impl.ResumeTransactionUtil contains +code derived from the Apache Flink project. + +The class com.hazelcast.internal.util.ConcurrentReferenceHashMap contains code written by Doug Lea +and updated within the WildFly project (https://github.com/wildfly/wildfly). + +The class org.apache.calcite.linq4j.tree.ConstantExpression contains code +originating from the Calcite project (https://github.com/apache/calcite). + +========================================================================= + diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-aspectj.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-aspectj.txt index 6a5ca86f5..4d4254f1d 100644 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-aspectj.txt +++ b/seatunnel-web-dist/release-docs/licenses/LICENSE-aspectj.txt @@ -1,71 +1,279 @@ +Per: https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt + Eclipse Public License - v 2.0 -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE (“AGREEMENT”). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 1. DEFINITIONS -“Contribution” means: -a) in the case of the initial Contributor, the initial content Distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are Distributed by that particular Contributor. A Contribution “originates” from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include changes or additions to the Program that are not Modified Works. -“Contributor” means any person or entity that Distributes the Program. +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. -“Licensed Patents” mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. +"Contributor" means any person or entity that Distributes the Program. -“Program” means the Contributions Distributed in accordance with this Agreement. +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. -“Recipient” means anyone who receives the Program under this Agreement or any Secondary License (as applicable), including Contributors. +"Program" means the Contributions Distributed in accordance with this +Agreement. -“Derivative Works” shall mean any work, whether in Source Code or other form, that is based on (or derived from) the Program and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. -“Modified Works” shall mean any work in Source Code or other form that results from an addition to, deletion from, or modification of the contents of the Program, including, for purposes of clarity any new file in Source Code form that contains any contents of the Program. Modified Works shall not include works that contain only declarations, interfaces, types, classes, structures, or files of the Program solely in each case in order to link to, bind by name, or subclass the Program or Modified Works thereof. +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. -“Distribute” means the acts of a) distributing or b) making available in any manner that enables the transfer of a copy. +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. -“Source Code” means the form of a Program preferred for making modifications, including but not limited to software source code, documentation source, and configuration files. +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. -“Secondary License” means either the GNU General Public License, Version 2.0, or any later versions of that license, including any exceptions or additional permissions as identified by the initial Contributor. +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. 2. GRANT OF RIGHTS -a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, Distribute and sublicense the Contribution of such Contributor, if any, and such Derivative Works. -b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in Source Code or other form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to Distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. -e) Notwithstanding the terms of any Secondary License, no Contributor makes additional grants to any Recipient (other than those set forth in this Agreement) as a result of such Recipient's receipt of the Program under the terms of a Secondary License (if permitted under the terms of Section 3). + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + 3. REQUIREMENTS + 3.1 If a Contributor Distributes the Program in any form, then: -a) the Program must also be made available as Source Code, in accordance with section 3.2, and the Contributor must accompany the Program with a statement that the Source Code for the Program is available under this Agreement, and informs Recipients how to obtain it in a reasonable manner on or through a medium customarily used for software exchange; and -b) the Contributor may Distribute the Program under a license different than this Agreement, provided that such license: -i) effectively disclaims on behalf of all other Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; -ii) effectively excludes on behalf of all other Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; -iii) does not attempt to limit or alter the recipients' rights in the Source Code under section 3.2; and -iv) requires any subsequent distribution of the Program by any party to be under a license that satisfies the requirements of this section 3. + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + 3.2 When the Program is Distributed as Source Code: -a) it must be made available under this Agreement, or if the Program (i) is combined with other material in a separate file or files made available under a Secondary License, and (ii) the initial Contributor attached to the Source Code the notice described in Exhibit A of this Agreement, then the Program may be made available under the terms of such Secondary Licenses, and -b) a copy of this Agreement must be included with each copy of the Program. -3.3 Contributors may not remove or alter any copyright, patent, trademark, attribution notices, disclaimers of warranty, or limitations of liability (‘notices’) contained within the Program from any copy of the Program which they Distribute, provided that Contributors may add their own appropriate notices. + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. 4. COMMERCIAL DISTRIBUTION -Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor (“Commercial Contributor”) hereby agrees to defend and indemnify every other Contributor (“Indemnified Contributor”) against any losses, damages and costs (collectively “Losses”) arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. -For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. 5. NO WARRANTY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. 6. DISCLAIMER OF LIABILITY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. 7. GENERAL -If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. -If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." -All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. -Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be Distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to Distribute the Program (including its Contributions) under the new version. + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. -Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. Nothing in this Agreement is intended to be enforceable by any entity that is not a Contributor or Recipient. No third-party beneficiary rights are created under this Agreement. \ No newline at end of file + You may add additional accurate notices of copyright ownership. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-javassist.html b/seatunnel-web-dist/release-docs/licenses/LICENSE-javassist.html deleted file mode 100644 index 66bcfb885..000000000 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-javassist.html +++ /dev/null @@ -1,373 +0,0 @@ - - -Javassist License - - - - -
MOZILLA PUBLIC LICENSE
Version -1.1 -

-

-
-

1. Definitions. -

    1.0.1. "Commercial Use" means distribution or otherwise making the - Covered Code available to a third party. -

    1.1. ''Contributor'' means each entity that creates or contributes - to the creation of Modifications. -

    1.2. ''Contributor Version'' means the combination of the Original - Code, prior Modifications used by a Contributor, and the Modifications made by - that particular Contributor. -

    1.3. ''Covered Code'' means the Original Code or Modifications or - the combination of the Original Code and Modifications, in each case including - portions thereof. -

    1.4. ''Electronic Distribution Mechanism'' means a mechanism - generally accepted in the software development community for the electronic - transfer of data. -

    1.5. ''Executable'' means Covered Code in any form other than Source - Code. -

    1.6. ''Initial Developer'' means the individual or entity identified - as the Initial Developer in the Source Code notice required by Exhibit - A. -

    1.7. ''Larger Work'' means a work which combines Covered Code or - portions thereof with code not governed by the terms of this License. -

    1.8. ''License'' means this document. -

    1.8.1. "Licensable" means having the right to grant, to the maximum - extent possible, whether at the time of the initial grant or subsequently - acquired, any and all of the rights conveyed herein. -

    1.9. ''Modifications'' means any addition to or deletion from the - substance or structure of either the Original Code or any previous - Modifications. When Covered Code is released as a series of files, a - Modification is: -

      A. Any addition to or deletion from the contents of a file - containing Original Code or previous Modifications. -

      B. Any new file that contains any part of the Original Code or - previous Modifications.
       

    1.10. ''Original Code'' - means Source Code of computer software code which is described in the Source - Code notice required by Exhibit A as Original Code, and which, at the - time of its release under this License is not already Covered Code governed by - this License. -

    1.10.1. "Patent Claims" means any patent claim(s), now owned or - hereafter acquired, including without limitation,  method, process, and - apparatus claims, in any patent Licensable by grantor. -

    1.11. ''Source Code'' means the preferred form of the Covered Code - for making modifications to it, including all modules it contains, plus any - associated interface definition files, scripts used to control compilation and - installation of an Executable, or source code differential comparisons against - either the Original Code or another well known, available Covered Code of the - Contributor's choice. The Source Code can be in a compressed or archival form, - provided the appropriate decompression or de-archiving software is widely - available for no charge. -

    1.12. "You'' (or "Your")  means an individual or a legal entity - exercising rights under, and complying with all of the terms of, this License - or a future version of this License issued under Section 6.1. For legal - entities, "You'' includes any entity which controls, is controlled by, or is - under common control with You. For purposes of this definition, "control'' - means (a) the power, direct or indirect, to cause the direction or management - of such entity, whether by contract or otherwise, or (b) ownership of more - than fifty percent (50%) of the outstanding shares or beneficial ownership of - such entity.

2. Source Code License. -
    2.1. The Initial Developer Grant.
    The Initial Developer hereby - grants You a world-wide, royalty-free, non-exclusive license, subject to third - party intellectual property claims: -
      (a)  under intellectual property rights (other than - patent or trademark) Licensable by Initial Developer to use, reproduce, - modify, display, perform, sublicense and distribute the Original Code (or - portions thereof) with or without Modifications, and/or as part of a Larger - Work; and -

      (b) under Patents Claims infringed by the making, using or selling - of Original Code, to make, have made, use, practice, sell, and offer for - sale, and/or otherwise dispose of the Original Code (or portions thereof). -

        -
        (c) the licenses granted in this Section 2.1(a) and (b) - are effective on the date Initial Developer first distributes Original Code - under the terms of this License. -

        (d) Notwithstanding Section 2.1(b) above, no patent license is - granted: 1) for code that You delete from the Original Code; 2) separate - from the Original Code;  or 3) for infringements caused by: i) the - modification of the Original Code or ii) the combination of the Original - Code with other software or devices.
         

      2.2. Contributor - Grant.
      Subject to third party intellectual property claims, each - Contributor hereby grants You a world-wide, royalty-free, non-exclusive - license -

        (a)  under intellectual property rights (other - than patent or trademark) Licensable by Contributor, to use, reproduce, - modify, display, perform, sublicense and distribute the Modifications - created by such Contributor (or portions thereof) either on an unmodified - basis, with other Modifications, as Covered Code and/or as part of a Larger - Work; and -

        (b) under Patent Claims infringed by the making, using, or selling - of  Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such - combination), to make, use, sell, offer for sale, have made, and/or - otherwise dispose of: 1) Modifications made by that Contributor (or portions - thereof); and 2) the combination of  Modifications made by that - Contributor with its Contributor Version (or portions of such - combination). -

        (c) the licenses granted in Sections 2.2(a) and 2.2(b) are - effective on the date Contributor first makes Commercial Use of the Covered - Code. -

        (d)    Notwithstanding Section 2.2(b) above, no - patent license is granted: 1) for any code that Contributor has deleted from - the Contributor Version; 2)  separate from the Contributor - Version;  3)  for infringements caused by: i) third party - modifications of Contributor Version or ii)  the combination of - Modifications made by that Contributor with other software  (except as - part of the Contributor Version) or other devices; or 4) under Patent Claims - infringed by Covered Code in the absence of Modifications made by that - Contributor.

    -


    3. Distribution Obligations. -

      3.1. Application of License.
      The Modifications which You create - or to which You contribute are governed by the terms of this License, - including without limitation Section 2.2. The Source Code version of - Covered Code may be distributed only under the terms of this License or a - future version of this License released under Section 6.1, and You must - include a copy of this License with every copy of the Source Code You - distribute. You may not offer or impose any terms on any Source Code version - that alters or restricts the applicable version of this License or the - recipients' rights hereunder. However, You may include an additional document - offering the additional rights described in Section 3.5. -

      3.2. Availability of Source Code.
      Any Modification which You - create or to which You contribute must be made available in Source Code form - under the terms of this License either on the same media as an Executable - version or via an accepted Electronic Distribution Mechanism to anyone to whom - you made an Executable version available; and if made available via Electronic - Distribution Mechanism, must remain available for at least twelve (12) months - after the date it initially became available, or at least six (6) months after - a subsequent version of that particular Modification has been made available - to such recipients. You are responsible for ensuring that the Source Code - version remains available even if the Electronic Distribution Mechanism is - maintained by a third party. -

      3.3. Description of Modifications.
      You must cause all Covered - Code to which You contribute to contain a file documenting the changes You - made to create that Covered Code and the date of any change. You must include - a prominent statement that the Modification is derived, directly or - indirectly, from Original Code provided by the Initial Developer and including - the name of the Initial Developer in (a) the Source Code, and (b) in any - notice in an Executable version or related documentation in which You describe - the origin or ownership of the Covered Code. -

      3.4. Intellectual Property Matters -

        (a) Third Party Claims.
        If Contributor has knowledge that a - license under a third party's intellectual property rights is required to - exercise the rights granted by such Contributor under Sections 2.1 or 2.2, - Contributor must include a text file with the Source Code distribution - titled "LEGAL'' which describes the claim and the party making the claim in - sufficient detail that a recipient will know whom to contact. If Contributor - obtains such knowledge after the Modification is made available as described - in Section 3.2, Contributor shall promptly modify the LEGAL file in all - copies Contributor makes available thereafter and shall take other steps - (such as notifying appropriate mailing lists or newsgroups) reasonably - calculated to inform those who received the Covered Code that new knowledge - has been obtained. -

        (b) Contributor APIs.
        If Contributor's Modifications include - an application programming interface and Contributor has knowledge of patent - licenses which are reasonably necessary to implement that API, Contributor - must also include this information in the LEGAL file. -
         

                - (c)    Representations. -
        Contributor represents that, except as disclosed pursuant to Section - 3.4(a) above, Contributor believes that Contributor's Modifications are - Contributor's original creation(s) and/or Contributor has sufficient rights - to grant the rights conveyed by this License.
      -


      3.5. Required Notices.
      You must duplicate the notice in - Exhibit A in each file of the Source Code.  If it is not possible - to put such notice in a particular Source Code file due to its structure, then - You must include such notice in a location (such as a relevant directory) - where a user would be likely to look for such a notice.  If You created - one or more Modification(s) You may add your name as a Contributor to the - notice described in Exhibit A.  You must also duplicate this - License in any documentation for the Source Code where You describe - recipients' rights or ownership rights relating to Covered Code.  You may - choose to offer, and to charge a fee for, warranty, support, indemnity or - liability obligations to one or more recipients of Covered Code. However, You - may do so only on Your own behalf, and not on behalf of the Initial Developer - or any Contributor. You must make it absolutely clear than any such warranty, - support, indemnity or liability obligation is offered by You alone, and You - hereby agree to indemnify the Initial Developer and every Contributor for any - liability incurred by the Initial Developer or such Contributor as a result of - warranty, support, indemnity or liability terms You offer. -

      3.6. Distribution of Executable Versions.
      You may distribute - Covered Code in Executable form only if the requirements of Section - 3.1-3.5 have been met for that Covered Code, and if You include a - notice stating that the Source Code version of the Covered Code is available - under the terms of this License, including a description of how and where You - have fulfilled the obligations of Section 3.2. The notice must be - conspicuously included in any notice in an Executable version, related - documentation or collateral in which You describe recipients' rights relating - to the Covered Code. You may distribute the Executable version of Covered Code - or ownership rights under a license of Your choice, which may contain terms - different from this License, provided that You are in compliance with the - terms of this License and that the license for the Executable version does not - attempt to limit or alter the recipient's rights in the Source Code version - from the rights set forth in this License. If You distribute the Executable - version under a different license You must make it absolutely clear that any - terms which differ from this License are offered by You alone, not by the - Initial Developer or any Contributor. You hereby agree to indemnify the - Initial Developer and every Contributor for any liability incurred by the - Initial Developer or such Contributor as a result of any such terms You offer. - -

      3.7. Larger Works.
      You may create a Larger Work by combining - Covered Code with other code not governed by the terms of this License and - distribute the Larger Work as a single product. In such a case, You must make - sure the requirements of this License are fulfilled for the Covered -Code.

    4. Inability to Comply Due to Statute or Regulation. -
      If it is impossible for You to comply with any of the terms of this - License with respect to some or all of the Covered Code due to statute, - judicial order, or regulation then You must: (a) comply with the terms of this - License to the maximum extent possible; and (b) describe the limitations and - the code they affect. Such description must be included in the LEGAL file - described in Section 3.4 and must be included with all distributions of - the Source Code. Except to the extent prohibited by statute or regulation, - such description must be sufficiently detailed for a recipient of ordinary - skill to be able to understand it.
    5. Application of this License. -
      This License applies to code to which the Initial Developer has attached - the notice in Exhibit A and to related Covered Code.
    6. Versions -of the License. -
      6.1. New Versions.
      Netscape Communications Corporation - (''Netscape'') may publish revised and/or new versions of the License from - time to time. Each version will be given a distinguishing version number. -

      6.2. Effect of New Versions.
      Once Covered Code has been - published under a particular version of the License, You may always continue - to use it under the terms of that version. You may also choose to use such - Covered Code under the terms of any subsequent version of the License - published by Netscape. No one other than Netscape has the right to modify the - terms applicable to Covered Code created under this License. -

      6.3. Derivative Works.
      If You create or use a modified version - of this License (which you may only do in order to apply it to code which is - not already Covered Code governed by this License), You must (a) rename Your - license so that the phrases ''Mozilla'', ''MOZILLAPL'', ''MOZPL'', - ''Netscape'', "MPL", ''NPL'' or any confusingly similar phrase do not appear - in your license (except to note that your license differs from this License) - and (b) otherwise make it clear that Your version of the license contains - terms which differ from the Mozilla Public License and Netscape Public - License. (Filling in the name of the Initial Developer, Original Code or - Contributor in the notice described in Exhibit A shall not of - themselves be deemed to be modifications of this License.)

    7. -DISCLAIMER OF WARRANTY. -
      COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS'' BASIS, WITHOUT - WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT - LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF DEFECTS, MERCHANTABLE, - FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE - QUALITY AND PERFORMANCE OF THE COVERED CODE IS WITH YOU. SHOULD ANY COVERED - CODE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY - OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR - CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS - LICENSE. NO USE OF ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS - DISCLAIMER.
    8. TERMINATION. -
      8.1.  This License and the rights granted hereunder will - terminate automatically if You fail to comply with terms herein and fail to - cure such breach within 30 days of becoming aware of the breach. All - sublicenses to the Covered Code which are properly granted shall survive any - termination of this License. Provisions which, by their nature, must remain in - effect beyond the termination of this License shall survive. -

      8.2.  If You initiate litigation by asserting a patent - infringement claim (excluding declatory judgment actions) against Initial - Developer or a Contributor (the Initial Developer or Contributor against whom - You file such action is referred to as "Participant")  alleging that: -

      (a)  such Participant's Contributor Version directly or - indirectly infringes any patent, then any and all rights granted by such - Participant to You under Sections 2.1 and/or 2.2 of this License shall, upon - 60 days notice from Participant terminate prospectively, unless if within 60 - days after receipt of notice You either: (i)  agree in writing to pay - Participant a mutually agreeable reasonable royalty for Your past and future - use of Modifications made by such Participant, or (ii) withdraw Your - litigation claim with respect to the Contributor Version against such - Participant.  If within 60 days of notice, a reasonable royalty and - payment arrangement are not mutually agreed upon in writing by the parties or - the litigation claim is not withdrawn, the rights granted by Participant to - You under Sections 2.1 and/or 2.2 automatically terminate at the expiration of - the 60 day notice period specified above. -

      (b)  any software, hardware, or device, other than such - Participant's Contributor Version, directly or indirectly infringes any - patent, then any rights granted to You by such Participant under Sections - 2.1(b) and 2.2(b) are revoked effective as of the date You first made, used, - sold, distributed, or had made, Modifications made by that Participant. -

      8.3.  If You assert a patent infringement claim against - Participant alleging that such Participant's Contributor Version directly or - indirectly infringes any patent where such claim is resolved (such as by - license or settlement) prior to the initiation of patent infringement - litigation, then the reasonable value of the licenses granted by such - Participant under Sections 2.1 or 2.2 shall be taken into account in - determining the amount or value of any payment or license. -

      8.4.  In the event of termination under Sections 8.1 or 8.2 - above,  all end user license agreements (excluding distributors and - resellers) which have been validly granted by You or any distributor hereunder - prior to termination shall survive termination.

    9. LIMITATION OF -LIABILITY. -
      UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING - NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY - OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, OR ANY SUPPLIER OF ANY - OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, - INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT - LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR - MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH - PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS - LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL - INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW - PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR - LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND - LIMITATION MAY NOT APPLY TO YOU.
    10. U.S. GOVERNMENT END USERS. -
      The Covered Code is a ''commercial item,'' as that term is defined in 48 - C.F.R. 2.101 (Oct. 1995), consisting of ''commercial computer software'' and - ''commercial computer software documentation,'' as such terms are used in 48 - C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. - 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users - acquire Covered Code with only those rights set forth herein.
    11. -MISCELLANEOUS. -
      This License represents the complete agreement concerning subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. This License shall be governed by California law provisions - (except to the extent applicable law, if any, provides otherwise), excluding - its conflict-of-law provisions. With respect to disputes in which at least one - party is a citizen of, or an entity chartered or registered to do business in - the United States of America, any litigation relating to this License shall be - subject to the jurisdiction of the Federal Courts of the Northern District of - California, with venue lying in Santa Clara County, California, with the - losing party responsible for costs, including without limitation, court costs - and reasonable attorneys' fees and expenses. The application of the United - Nations Convention on Contracts for the International Sale of Goods is - expressly excluded. Any law or regulation which provides that the language of - a contract shall be construed against the drafter shall not apply to this - License.
    12. RESPONSIBILITY FOR CLAIMS. -
      As between Initial Developer and the Contributors, each party is - responsible for claims and damages arising, directly or indirectly, out of its - utilization of rights under this License and You agree to work with Initial - Developer and Contributors to distribute such responsibility on an equitable - basis. Nothing herein is intended or shall be deemed to constitute any - admission of liability.
    13. MULTIPLE-LICENSED CODE. -
      Initial Developer may designate portions of the Covered Code as - "Multiple-Licensed".  "Multiple-Licensed" means that the Initial - Developer permits you to utilize portions of the Covered Code under Your - choice of the MPL or the alternative licenses, if any, specified by the - Initial Developer in the file described in Exhibit A.
    -


    EXHIBIT A -Mozilla Public License. -

      The contents of this file are subject to the Mozilla Public License - Version 1.1 (the "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at -
      http://www.mozilla.org/MPL/ -

      Software distributed under the License is distributed on an "AS IS" basis, - WITHOUT WARRANTY OF
      ANY KIND, either express or implied. See the License - for the specific language governing rights and
      limitations under the - License. -

      The Original Code is Javassist. -

      The Initial Developer of the Original Code is Shigeru Chiba. - Portions created by the Initial Developer are
        - Copyright (C) 1999- Shigeru Chiba. All Rights Reserved. -

      Contributor(s): __Bill Burke, Jason T. Greene______________. - -

      Alternatively, the contents of this software may be used under the -terms of the GNU Lesser General Public License Version 2.1 or later -(the "LGPL"), or the Apache License Version 2.0 (the "AL"), -in which case the provisions of the LGPL or the AL are applicable -instead of those above. If you wish to allow use of your version of -this software only under the terms of either the LGPL or the AL, and not to allow others to -use your version of this software under the terms of the MPL, indicate -your decision by deleting the provisions above and replace them with -the notice and other provisions required by the LGPL or the AL. If you do not -delete the provisions above, a recipient may use your version of this -software under the terms of any one of the MPL, the LGPL or the AL. - -

    - - \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-javax-annotation.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-javax-annotation.txt deleted file mode 100644 index a0ccc9356..000000000 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-javax-annotation.txt +++ /dev/null @@ -1,263 +0,0 @@ -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 - -1. Definitions. - - 1.1. Contributor. means each individual or entity that creates or contributes to the creation of Modifications. - - 1.2. Contributor Version. means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. - - 1.3. Covered Software. means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. - - 1.4. Executable. means the Covered Software in any form other than Source Code. - - 1.5. Initial Developer. means the individual or entity that first makes Original Software available under this License. - - 1.6. Larger Work. means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. - - 1.7. License. means this document. - - 1.8. Licensable. means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. - - 1.9. Modifications. means the Source Code and Executable form of any of the following: - - A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; - - B. Any new file that contains any part of the Original Software or previous Modification; or - - C. Any new file that is contributed or otherwise made available under the terms of this License. - - 1.10. Original Software. means the Source Code and Executable form of computer software code that is originally released under this License. - - 1.11. Patent Claims. means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. - - 1.12. Source Code. means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. - - 1.13. You. (or .Your.) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, .You. includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, .control. means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants. - - 2.1. The Initial Developer Grant. - - Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: - - (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). - - (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. - - (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. - - 2.2. Contributor Grant. - - Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: - - (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). - - (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. - - (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. - -3. Distribution Obligations. - - 3.1. Availability of Source Code. - Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. - - 3.2. Modifications. - The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. - - 3.3. Required Notices. - You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. - - 3.4. Application of Additional Terms. - You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients. rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. - - 3.5. Distribution of Executable Versions. - You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient.s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. - - 3.6. Larger Works. - You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. - -4. Versions of the License. - - 4.1. New Versions. - Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. - - 4.2. Effect of New Versions. - You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. - - 4.3. Modified Versions. - When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. - -5. DISCLAIMER OF WARRANTY. - - COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN .AS IS. BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - - 6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. - - 6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as .Participant.) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. - - 6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - - UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY.S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. - - The Covered Software is a .commercial item,. as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of .commercial computer software. (as that term is defined at 48 C.F.R. ? 252.227-7014(a)(1)) and .commercial computer software documentation. as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. - -9. MISCELLANEOUS. - - This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction.s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys. fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - - As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. - - NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) - - The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. - - -The GNU General Public License (GPL) Version 2, June 1991 - - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification follow. - - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. - - c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. - -3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL VERSION 2 - -Certain source files distributed by Sun Microsystems, Inc. are subject to the following clarification and special exception to the GPL Version 2, but only where Sun has expressly included in the particular source file's header the words - -"Sun designates this particular file as subject to the "Classpath" exception as provided by Sun in the License file that accompanied this code." - -Linking this library statically or dynamically with other modules is making a combined work based on this library. Thus, the terms and conditions of the GNU General Public License Version 2 cover the whole combination. - -As a special exception, the copyright holders of this library give you permission to link this library with independent modules to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions of the license of that module.? An independent module is a module which is not derived from or based on this library.? If you modify this library, you may extend this exception to your version of the library, but you are not obligated to do so.? If you do not wish to do so, delete this exception statement from your version. diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-javax.el.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-javax.el.txt deleted file mode 100644 index 081ea0ce1..000000000 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-javax.el.txt +++ /dev/null @@ -1,759 +0,0 @@ -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1 - -1. Definitions. - - 1.1. "Contributor" means each individual or entity that creates or - contributes to the creation of Modifications. - - 1.2. "Contributor Version" means the combination of the Original - Software, prior Modifications used by a Contributor (if any), and - the Modifications made by that particular Contributor. - - 1.3. "Covered Software" means (a) the Original Software, or (b) - Modifications, or (c) the combination of files containing Original - Software with files containing Modifications, in each case including - portions thereof. - - 1.4. "Executable" means the Covered Software in any form other than - Source Code. - - 1.5. "Initial Developer" means the individual or entity that first - makes Original Software available under this License. - - 1.6. "Larger Work" means a work which combines Covered Software or - portions thereof with code not governed by the terms of this License. - - 1.7. "License" means this document. - - 1.8. "Licensable" means having the right to grant, to the maximum - extent possible, whether at the time of the initial grant or - subsequently acquired, any and all of the rights conveyed herein. - - 1.9. "Modifications" means the Source Code and Executable form of - any of the following: - - A. Any file that results from an addition to, deletion from or - modification of the contents of a file containing Original Software - or previous Modifications; - - B. Any new file that contains any part of the Original Software or - previous Modification; or - - C. Any new file that is contributed or otherwise made available - under the terms of this License. - - 1.10. "Original Software" means the Source Code and Executable form - of computer software code that is originally released under this - License. - - 1.11. "Patent Claims" means any patent claim(s), now owned or - hereafter acquired, including without limitation, method, process, - and apparatus claims, in any patent Licensable by grantor. - - 1.12. "Source Code" means (a) the common form of computer software - code in which modifications are made and (b) associated - documentation included in or with such code. - - 1.13. "You" (or "Your") means an individual or a legal entity - exercising rights under, and complying with all of the terms of, - this License. For legal entities, "You" includes any entity which - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants. - - 2.1. The Initial Developer Grant. - - Conditioned upon Your compliance with Section 3.1 below and subject - to third party intellectual property claims, the Initial Developer - hereby grants You a world-wide, royalty-free, non-exclusive license: - - (a) under intellectual property rights (other than patent or - trademark) Licensable by Initial Developer, to use, reproduce, - modify, display, perform, sublicense and distribute the Original - Software (or portions thereof), with or without Modifications, - and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using or selling of - Original Software, to make, have made, use, practice, sell, and - offer for sale, and/or otherwise dispose of the Original Software - (or portions thereof). - - (c) The licenses granted in Sections 2.1(a) and (b) are effective on - the date Initial Developer first distributes or otherwise makes the - Original Software available to a third party under the terms of this - License. - - (d) Notwithstanding Section 2.1(b) above, no patent license is - granted: (1) for code that You delete from the Original Software, or - (2) for infringements caused by: (i) the modification of the - Original Software, or (ii) the combination of the Original Software - with other software or devices. - - 2.2. Contributor Grant. - - Conditioned upon Your compliance with Section 3.1 below and subject - to third party intellectual property claims, each Contributor hereby - grants You a world-wide, royalty-free, non-exclusive license: - - (a) under intellectual property rights (other than patent or - trademark) Licensable by Contributor to use, reproduce, modify, - display, perform, sublicense and distribute the Modifications - created by such Contributor (or portions thereof), either on an - unmodified basis, with other Modifications, as Covered Software - and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using, or selling - of Modifications made by that Contributor either alone and/or in - combination with its Contributor Version (or portions of such - combination), to make, use, sell, offer for sale, have made, and/or - otherwise dispose of: (1) Modifications made by that Contributor (or - portions thereof); and (2) the combination of Modifications made by - that Contributor with its Contributor Version (or portions of such - combination). - - (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective - on the date Contributor first distributes or otherwise makes the - Modifications available to a third party. - - (d) Notwithstanding Section 2.2(b) above, no patent license is - granted: (1) for any code that Contributor has deleted from the - Contributor Version; (2) for infringements caused by: (i) third - party modifications of Contributor Version, or (ii) the combination - of Modifications made by that Contributor with other software - (except as part of the Contributor Version) or other devices; or (3) - under Patent Claims infringed by Covered Software in the absence of - Modifications made by that Contributor. - -3. Distribution Obligations. - - 3.1. Availability of Source Code. - - Any Covered Software that You distribute or otherwise make available - in Executable form must also be made available in Source Code form - and that Source Code form must be distributed only under the terms - of this License. You must include a copy of this License with every - copy of the Source Code form of the Covered Software You distribute - or otherwise make available. You must inform recipients of any such - Covered Software in Executable form as to how they can obtain such - Covered Software in Source Code form in a reasonable manner on or - through a medium customarily used for software exchange. - - 3.2. Modifications. - - The Modifications that You create or to which You contribute are - governed by the terms of this License. You represent that You - believe Your Modifications are Your original creation(s) and/or You - have sufficient rights to grant the rights conveyed by this License. - - 3.3. Required Notices. - - You must include a notice in each of Your Modifications that - identifies You as the Contributor of the Modification. You may not - remove or alter any copyright, patent or trademark notices contained - within the Covered Software, or any notices of licensing or any - descriptive text giving attribution to any Contributor or the - Initial Developer. - - 3.4. Application of Additional Terms. - - You may not offer or impose any terms on any Covered Software in - Source Code form that alters or restricts the applicable version of - this License or the recipients' rights hereunder. You may choose to - offer, and to charge a fee for, warranty, support, indemnity or - liability obligations to one or more recipients of Covered Software. - However, you may do so only on Your own behalf, and not on behalf of - the Initial Developer or any Contributor. You must make it - absolutely clear that any such warranty, support, indemnity or - liability obligation is offered by You alone, and You hereby agree - to indemnify the Initial Developer and every Contributor for any - liability incurred by the Initial Developer or such Contributor as a - result of warranty, support, indemnity or liability terms You offer. - - 3.5. Distribution of Executable Versions. - - You may distribute the Executable form of the Covered Software under - the terms of this License or under the terms of a license of Your - choice, which may contain terms different from this License, - provided that You are in compliance with the terms of this License - and that the license for the Executable form does not attempt to - limit or alter the recipient's rights in the Source Code form from - the rights set forth in this License. If You distribute the Covered - Software in Executable form under a different license, You must make - it absolutely clear that any terms which differ from this License - are offered by You alone, not by the Initial Developer or - Contributor. You hereby agree to indemnify the Initial Developer and - every Contributor for any liability incurred by the Initial - Developer or such Contributor as a result of any such terms You offer. - - 3.6. Larger Works. - - You may create a Larger Work by combining Covered Software with - other code not governed by the terms of this License and distribute - the Larger Work as a single product. In such a case, You must make - sure the requirements of this License are fulfilled for the Covered - Software. - -4. Versions of the License. - - 4.1. New Versions. - - Oracle is the initial license steward and may publish revised and/or - new versions of this License from time to time. Each version will be - given a distinguishing version number. Except as provided in Section - 4.3, no one other than the license steward has the right to modify - this License. - - 4.2. Effect of New Versions. - - You may always continue to use, distribute or otherwise make the - Covered Software available under the terms of the version of the - License under which You originally received the Covered Software. If - the Initial Developer includes a notice in the Original Software - prohibiting it from being distributed or otherwise made available - under any subsequent version of the License, You must distribute and - make the Covered Software available under the terms of the version - of the License under which You originally received the Covered - Software. Otherwise, You may also choose to use, distribute or - otherwise make the Covered Software available under the terms of any - subsequent version of the License published by the license steward. - - 4.3. Modified Versions. - - When You are an Initial Developer and You want to create a new - license for Your Original Software, You may create and use a - modified version of this License if You: (a) rename the license and - remove any references to the name of the license steward (except to - note that the license differs from this License); and (b) otherwise - make it clear that the license contains terms which differ from this - License. - -5. DISCLAIMER OF WARRANTY. - - COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, - WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, - INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE - IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR - NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF - THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE - DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY - OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, - REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN - ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS - AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - - 6.1. This License and the rights granted hereunder will terminate - automatically if You fail to comply with terms herein and fail to - cure such breach within 30 days of becoming aware of the breach. - Provisions which, by their nature, must remain in effect beyond the - termination of this License shall survive. - - 6.2. If You assert a patent infringement claim (excluding - declaratory judgment actions) against Initial Developer or a - Contributor (the Initial Developer or Contributor against whom You - assert such claim is referred to as "Participant") alleging that the - Participant Software (meaning the Contributor Version where the - Participant is a Contributor or the Original Software where the - Participant is the Initial Developer) directly or indirectly - infringes any patent, then any and all rights granted directly or - indirectly to You by such Participant, the Initial Developer (if the - Initial Developer is not the Participant) and all Contributors under - Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice - from Participant terminate prospectively and automatically at the - expiration of such 60 day notice period, unless if within such 60 - day period You withdraw Your claim with respect to the Participant - Software against such Participant either unilaterally or pursuant to - a written agreement with Participant. - - 6.3. If You assert a patent infringement claim against Participant - alleging that the Participant Software directly or indirectly - infringes any patent where such claim is resolved (such as by - license or settlement) prior to the initiation of patent - infringement litigation, then the reasonable value of the licenses - granted by such Participant under Sections 2.1 or 2.2 shall be taken - into account in determining the amount or value of any payment or - license. - - 6.4. In the event of termination under Sections 6.1 or 6.2 above, - all end user licenses that have been validly granted by You or any - distributor hereunder prior to termination (excluding licenses - granted to You by any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - - UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT - (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE - INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF - COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE - TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR - CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT - LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER - FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR - LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE - POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT - APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH - PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH - LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR - LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION - AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. - - The Covered Software is a "commercial item," as that term is defined - in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer - software" (as that term is defined at 48 C.F.R. - 252.227-7014(a)(1)) and "commercial computer software documentation" - as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent - with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 - (June 1995), all U.S. Government End Users acquire Covered Software - with only those rights set forth herein. This U.S. Government Rights - clause is in lieu of, and supersedes, any other FAR, DFAR, or other - clause or provision that addresses Government rights in computer - software under this License. - -9. MISCELLANEOUS. - - This License represents the complete agreement concerning subject - matter hereof. If any provision of this License is held to be - unenforceable, such provision shall be reformed only to the extent - necessary to make it enforceable. This License shall be governed by - the law of the jurisdiction specified in a notice contained within - the Original Software (except to the extent applicable law, if any, - provides otherwise), excluding such jurisdiction's conflict-of-law - provisions. Any litigation relating to this License shall be subject - to the jurisdiction of the courts located in the jurisdiction and - venue specified in a notice contained within the Original Software, - with the losing party responsible for costs, including, without - limitation, court costs and reasonable attorneys' fees and expenses. - The application of the United Nations Convention on Contracts for - the International Sale of Goods is expressly excluded. Any law or - regulation which provides that the language of a contract shall be - construed against the drafter shall not apply to this License. You - agree that You alone are responsible for compliance with the United - States export administration regulations (and the export control - laws and regulation of any other countries) when You use, distribute - or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - - As between Initial Developer and the Contributors, each party is - responsible for claims and damages arising, directly or indirectly, - out of its utilization of rights under this License and You agree to - work with Initial Developer and Contributors to distribute such - responsibility on an equitable basis. Nothing herein is intended or - shall be deemed to constitute any admission of liability. - ------------------------------------------------------------------------- - -NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION -LICENSE (CDDL) - -The code released under the CDDL shall be governed by the laws of the -State of California (excluding conflict-of-law provisions). Any -litigation relating to this License shall be subject to the jurisdiction -of the Federal Courts of the Northern District of California and the -state courts of the State of California, with venue lying in Santa Clara -County, California. - - - - The GNU General Public License (GPL) Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -51 Franklin Street, Fifth Floor -Boston, MA 02110-1335 -USA - -Everyone is permitted to copy and distribute verbatim copies -of this license document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to -share and change it. By contrast, the GNU General Public License is -intended to guarantee your freedom to share and change free software--to -make sure the software is free for all its users. This General Public -License applies to most of the Free Software Foundation's software and -to any other program whose authors commit to using it. (Some other Free -Software Foundation software is covered by the GNU Library General -Public License instead.) You can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. -Our General Public Licenses are designed to make sure that you have the -freedom to distribute copies of free software (and charge for this -service if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs; and that you know you can do these things. - -To protect your rights, we need to make restrictions that forbid anyone -to deny you these rights or to ask you to surrender the rights. These -restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis -or for a fee, you must give the recipients all the rights that you have. -You must make sure that they, too, receive or can get the source code. -And you must show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - -Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - -Finally, any free program is threatened constantly by software patents. -We wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program -proprietary. To prevent this, we have made it clear that any patent must -be licensed for everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and -modification follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a -notice placed by the copyright holder saying it may be distributed under -the terms of this General Public License. The "Program", below, refers -to any such program or work, and a "work based on the Program" means -either the Program or any derivative work under copyright law: that is -to say, a work containing the Program or a portion of it, either -verbatim or with modifications and/or translated into another language. -(Hereinafter, translation is included without limitation in the term -"modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of running -the Program is not restricted, and the output from the Program is -covered only if its contents constitute a work based on the Program -(independent of having been made by running the Program). Whether that -is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source -code as you receive it, in any medium, provided that you conspicuously -and appropriately publish on each copy an appropriate copyright notice -and disclaimer of warranty; keep intact all the notices that refer to -this License and to the absence of any warranty; and give any other -recipients of the Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of -it, thus forming a work based on the Program, and copy and distribute -such modifications or work under the terms of Section 1 above, provided -that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any part - thereof, to be licensed as a whole at no charge to all third parties - under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a notice - that there is no warranty (or else, saying that you provide a - warranty) and that users may redistribute the program under these - conditions, and telling the user how to view a copy of this License. - (Exception: if the Program itself is interactive but does not - normally print such an announcement, your work based on the Program - is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, and -can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based on -the Program, the distribution of the whole must be on the terms of this -License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of a -storage or distribution medium does not bring the other work under the -scope of this License. - -3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your cost - of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed - only for noncommercial distribution and only if you received the - program in object code or executable form with such an offer, in - accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source code -means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to control -compilation and installation of the executable. However, as a special -exception, the source code distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies the -executable. - -If distribution of executable or object code is made by offering access -to copy from a designated place, then offering equivalent access to copy -the source code from the same place counts as distribution of the source -code, even though third parties are not compelled to copy the source -along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt otherwise -to copy, modify, sublicense or distribute the Program is void, and will -automatically terminate your rights under this License. However, parties -who have received copies, or rights, from you under this License will -not have their licenses terminated so long as such parties remain in -full compliance. - -5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and all -its terms and conditions for copying, distributing or modifying the -Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further restrictions -on the recipients' exercise of the rights granted herein. You are not -responsible for enforcing compliance by third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot distribute -so as to satisfy simultaneously your obligations under this License and -any other pertinent obligations, then as a consequence you may not -distribute the Program at all. For example, if a patent license would -not permit royalty-free redistribution of the Program by all those who -receive copies directly or indirectly through you, then the only way you -could satisfy both it and this License would be to refrain entirely from -distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is implemented -by public license practices. Many people have made generous -contributions to the wide range of software distributed through that -system in reliance on consistent application of that system; it is up to -the author/donor to decide if he or she is willing to distribute -software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be -a consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License may -add an explicit geographical distribution limitation excluding those -countries, so that distribution is permitted only in or among countries -not thus excluded. In such case, this License incorporates the -limitation as if written in the body of this License. - -9. The Free Software Foundation may publish revised and/or new -versions of the General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Program does not specify a version -number of this License, you may choose any version ever published by the -Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the -author to ask for permission. For software which is copyrighted by the -Free Software Foundation, write to the Free Software Foundation; we -sometimes make exceptions for this. Our decision will be guided by the -two goals of preserving the free status of all derivatives of our free -software and of promoting the sharing and reuse of software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, -EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE -ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH -YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL -NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR -DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL -DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM -(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED -INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF -THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR -OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to -attach them to the start of each source file to most effectively convey -the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type - `show w'. This is free software, and you are welcome to redistribute - it under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the -appropriate parts of the General Public License. Of course, the commands -you use may be called something other than `show w' and `show c'; they -could even be mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - program `Gnomovision' (which makes passes at compilers) written by - James Hacker. - - signature of Ty Coon, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications -with the library. If this is what you want to do, use the GNU Library -General Public License instead of this License. - -# - -Certain source files distributed by Oracle America, Inc. and/or its -affiliates are subject to the following clarification and special -exception to the GPLv2, based on the GNU Project exception for its -Classpath libraries, known as the GNU Classpath Exception, but only -where Oracle has expressly included in the particular source file's -header the words "Oracle designates this particular file as subject to -the "Classpath" exception as provided by Oracle in the LICENSE file -that accompanied this code." - -You should also note that Oracle includes multiple, independent -programs in this software package. Some of those programs are provided -under licenses deemed incompatible with the GPLv2 by the Free Software -Foundation and others. For example, the package includes programs -licensed under the Apache License, Version 2.0. Such programs are -licensed to you under their original licenses. - -Oracle facilitates your further distribution of this package by adding -the Classpath Exception to the necessary parts of its GPLv2 code, which -permits you to use that code in combination with other independent -modules not licensed under the GPLv2. However, note that this would -not permit you to commingle code under an incompatible license with -Oracle's GPLv2 licensed code by, for example, cutting and pasting such -code into a file also containing Oracle's GPLv2 licensed code and then -distributing the result. Additionally, if you were to remove the -Classpath Exception from any of the files to which it applies and -distribute the result, you would likely be required to license some or -all of the other code in that distribution under the GPLv2 as well, and -since the GPLv2 is incompatible with the license terms of some items -included in the distribution by Oracle, removing the Classpath -Exception could therefore effectively compromise your ability to -further distribute the package. - -Proceed with caution and we recommend that you obtain the advice of a -lawyer skilled in open source matters before removing the Classpath -Exception or making modifications to this package which may -subsequently be redistributed and/or involve the use of third party -software. - -CLASSPATH EXCEPTION -Linking this library statically or dynamically with other modules is -making a combined work based on this library. Thus, the terms and -conditions of the GNU General Public License version 2 cover the whole -combination. - -As a special exception, the copyright holders of this library give you -permission to link this library with independent modules to produce an -executable, regardless of the license terms of these independent -modules, and to copy and distribute the resulting executable under -terms of your choice, provided that you also meet, for each linked -independent module, the terms and conditions of the license of that -module. An independent module is a module which is not derived from or -based on this library. If you modify this library, you may extend this -exception to your version of the library, but you are not obligated to -do so. If you do not wish to do so, delete this exception statement -from your version. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-cron-utils.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-framework.txt similarity index 72% rename from seatunnel-web-dist/release-docs/licenses/LICENSE-cron-utils.txt rename to seatunnel-web-dist/release-docs/licenses/LICENSE-spring-framework.txt index 7a4a3ea24..2221e3246 100644 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-cron-utils.txt +++ b/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-framework.txt @@ -1,7 +1,6 @@ - Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -193,10 +192,98 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. + +======================================================================= + +SPRING FRAMEWORK 5.3.20 SUBCOMPONENTS: + +Spring Framework 5.3.20 includes a number of subcomponents +with separate copyright notices and license terms. The product that +includes this file does not necessarily use all the open source +subcomponents referred to below. Your use of the source +code for these subcomponents is subject to the terms and +conditions of the following licenses. + + +>>> ASM 9.1 (org.ow2.asm:asm:9.1, org.ow2.asm:asm-commons:9.1): + +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +Copyright (c) 1999-2009, OW2 Consortium + + +>>> CGLIB 3.3 (cglib:cglib:3.3): + +Per the LICENSE file in the CGLIB JAR distribution downloaded from +https://github.com/cglib/cglib/releases/download/RELEASE_3_3_0/cglib-3.3.0.jar, +CGLIB 3.3 is licensed under the Apache License, version 2.0, the text of which +is included above. + + +>>> Objenesis 3.2 (org.objenesis:objenesis:3.2): + +Per the LICENSE file in the Objenesis ZIP distribution downloaded from +http://objenesis.org/download.html, Objenesis 3.2 is licensed under the +Apache License, version 2.0, the text of which is included above. + +Per the NOTICE file in the Objenesis ZIP distribution downloaded from +http://objenesis.org/download.html and corresponding to section 4d of the +Apache License, Version 2.0, in this case for Objenesis: + +Objenesis +Copyright 2006-2019 Joe Walnes, Henri Tremblay, Leonardo Mesquita + + +=============================================================================== + +To the extent any open source components are licensed under the EPL and/or +other similar licenses that require the source code and/or modifications to +source code to be made available (as would be noted above), you may obtain a +copy of the source code corresponding to the binaries for such open source +components and modifications thereto, if any, (the "Source Files"), by +downloading the Source Files from https://spring.io/projects, Pivotal's website +at https://network.pivotal.io/open-source, or by sending a request, with your +name and address to: Pivotal Software, Inc., 875 Howard Street, 5th floor, San +Francisco, CA 94103, Attention: General Counsel. All such requests should +clearly specify: OPEN SOURCE FILES REQUEST, Attention General Counsel. Pivotal +can mail a copy of the Source Files to you on a CD or equivalent physical +medium. + +This offer to obtain a copy of the Source Files is valid for three years from +the date you acquired this Software product. Alternatively, the Source Files +may accompany the Software. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-plugin.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-plugin.txt index 220e23c80..7b1adee7a 100644 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-plugin.txt +++ b/seatunnel-web-dist/release-docs/licenses/LICENSE-spring-plugin.txt @@ -1,6 +1,6 @@ - Apache License + Apache License Version 2.0, January 2004 - https://www.apache.org/licenses/ + http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -192,7 +192,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -207,7 +207,7 @@ similar licenses that require the source code and/or modifications to source code to be made available (as would be noted above), you may obtain a copy of the source code corresponding to the binaries for such open source components and modifications thereto, if any, (the "Source Files"), by -downloading the Source Files from https://www.springsource.org/download, +downloading the Source Files from http://www.springsource.org/download, or by sending a request, with your name and address to: VMware, Inc., 3401 Hillview Avenue, Palo Alto, CA 94304, United States of America or email info@vmware.com. All such requests should clearly specify: OPEN SOURCE FILES REQUEST, Attention General diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core.txt b/seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core-1.5.0.txt similarity index 94% rename from seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core.txt rename to seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core-1.5.0.txt index e28001318..b89e980f8 100644 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core.txt +++ b/seatunnel-web-dist/release-docs/licenses/LICENSE-swagger-core-1.5.0.txt @@ -1,4 +1,4 @@ -Copyright 2016 SmartBear Software +Copyright 2015 SmartBear Software Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/seatunnel-web-dist/release-docs/licenses/LICENSE-jsoup.txt b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-layout similarity index 92% rename from seatunnel-web-dist/release-docs/licenses/LICENSE-jsoup.txt rename to seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-layout index d704b1265..202f17d90 100644 --- a/seatunnel-web-dist/release-docs/licenses/LICENSE-jsoup.txt +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-layout @@ -1,6 +1,6 @@ -The MIT License +MIT License -Copyright (c) 2009-2021 Jonathan Hedley +Copyright (c) 2018 Alipay.inc Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6 b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6 new file mode 100644 index 000000000..3d63a420a --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6 @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021-2022 Alipay.inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6-vue-shape b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6-vue-shape new file mode 100644 index 000000000..e0a7ec077 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@antv-x6-vue-shape @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021-2023 Alipay.inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@vueuse-core b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@vueuse-core new file mode 100644 index 000000000..b2501586b --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-@vueuse-core @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019-PRESENT Anthony Fu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-autoprefixer b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-autoprefixer new file mode 100644 index 000000000..1b21e2bc5 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-autoprefixer @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright 2013 Andrey Sitnik + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-axios b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-axios new file mode 100644 index 000000000..2d8d66aa4 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-axios @@ -0,0 +1,19 @@ +Copyright (c) 2014-present Matt Zabriskie + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns new file mode 100644 index 000000000..e3f00547e --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Sasha Koss and Lesha Koss https://kossnocorp.mit-license.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns-tz b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns-tz new file mode 100644 index 000000000..640ef802c --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-date-fns-tz @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright © 2018 Marnus Weststrate + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-lodash b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-lodash new file mode 100644 index 000000000..5b807415b --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-lodash @@ -0,0 +1,49 @@ +The MIT License + +Copyright JS Foundation and other contributors + +Based on Underscore.js, copyright Jeremy Ashkenas, +DocumentCloud and Investigative Reporters & Editors + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/lodash/lodash + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +==== + +Copyright and related rights for sample code are waived via CC0. Sample +code is defined as all source code displayed within the prose of the +documentation. + +CC0: http://creativecommons.org/publicdomain/zero/1.0/ + +==== + +Files located in the node_modules and vendor directories are externally +maintained libraries used by this software which have their own +licenses; we recommend you read them, as their terms may differ from the +terms above. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-monaco-editor b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-monaco-editor new file mode 100644 index 000000000..f69246720 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-monaco-editor @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 - present Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-naive-ui b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-naive-ui new file mode 100644 index 000000000..b75aa683c --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-naive-ui @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 TuSimple + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-nprogress b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-nprogress new file mode 100644 index 000000000..0ab1fefd9 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-nprogress @@ -0,0 +1,19 @@ +Copyright (c) 2013-2014 Rico Sta. Cruz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia new file mode 100644 index 000000000..6e1123909 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019-present Eduardo San Martin Morote + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia-plugin-persistedstate b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia-plugin-persistedstate new file mode 100644 index 000000000..69d314b8f --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-pinia-plugin-persistedstate @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Sacha 'PraZ' Bouillez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-postcss b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-postcss new file mode 100644 index 000000000..1b21e2bc5 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-postcss @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright 2013 Andrey Sitnik + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-screenfull b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-screenfull new file mode 100644 index 000000000..0b7d44300 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-screenfull @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in the +Software without restriction, including without limitation the rights to use, copy, +modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-tailwindcss b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-tailwindcss new file mode 100644 index 000000000..2ba9d4cd1 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-tailwindcss @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Tailwind Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue new file mode 100644 index 000000000..04600aaba --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018-present, Yuxi (Evan) You + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-i18n b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-i18n new file mode 100644 index 000000000..e28fb1a73 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-i18n @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2016 kazuya kawaguchi + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-router b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-router new file mode 100644 index 000000000..8a8c7c249 --- /dev/null +++ b/seatunnel-web-dist/release-docs/licenses/ui-license/LICENSE-vue-router @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-present Evan You + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/seatunnel-web-dist/src/main/assembly/assembly-src.xml b/seatunnel-web-dist/src/main/assembly/assembly-src.xml index 7f053147e..bbcc9e889 100644 --- a/seatunnel-web-dist/src/main/assembly/assembly-src.xml +++ b/seatunnel-web-dist/src/main/assembly/assembly-src.xml @@ -50,6 +50,12 @@ release.properties **/pom.xml.releaseBackup *.gpg + + + **/seatunnel-ui/dist/** + **/seatunnel-ui/node/** + **/seatunnel-ui/node_modules/** + **/.github/** **/.dlc.json diff --git a/seatunnel-web-dist/src/main/assembly/seatunnel-web-ci.xml b/seatunnel-web-dist/src/main/assembly/seatunnel-web-ci.xml index fc041ea76..f7fb910d8 100644 --- a/seatunnel-web-dist/src/main/assembly/seatunnel-web-ci.xml +++ b/seatunnel-web-dist/src/main/assembly/seatunnel-web-ci.xml @@ -71,7 +71,7 @@ - + false true @@ -117,7 +117,6 @@ ch.qos.logback:logback-core:jar - false diff --git a/seatunnel-web-dist/src/main/assembly/seatunnel-web.xml b/seatunnel-web-dist/src/main/assembly/seatunnel-web.xml index 3cb18a959..7d020467b 100644 --- a/seatunnel-web-dist/src/main/assembly/seatunnel-web.xml +++ b/seatunnel-web-dist/src/main/assembly/seatunnel-web.xml @@ -25,6 +25,7 @@ tar.gz true + ${project.build.finalName}-bin ../seatunnel-server/seatunnel-app/src/main/resources @@ -71,22 +72,6 @@ - - - false - true - false - - *:*:jar - - - org.apache.seatunnel:datasource-*:jar - org.apache.logging.log4j:*:jar - - /libs - provided - - false true diff --git a/tools/dependencies/checkLicense.sh b/tools/dependencies/checkLicense.sh index 7de125e95..5b7034210 100755 --- a/tools/dependencies/checkLicense.sh +++ b/tools/dependencies/checkLicense.sh @@ -23,7 +23,7 @@ if [ -d "/tmp/seatunnel-dependencies" ]; then rm -rf /tmp/seatunnel-dependencies/* fi -./mvnw clean -pl '!seatunnel-web-dist' --batch-mode --no-snapshot-updates dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies +./mvnw clean -Prelease --batch-mode --no-snapshot-updates dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies # List all modules(jars) that belong to the SeaTunnel itself, these will be ignored when checking the dependency ls /tmp/seatunnel-dependencies | sort > all-dependencies.txt diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 797b0980f..1387d7807 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -3,16 +3,15 @@ aspectjweaver-1.9.7.jar classmate-1.3.1.jar commons-collections4-4.4.jar commons-codec-1.13.jar -commons-lang3-3.12.0.jar commons-io-2.11.0.jar config-1.3.3.jar -cron-utils-9.1.6.jar +db2jcc-db2jcc4.jar gson-2.8.6.jar guava-19.0.jar hibernate-validator-6.2.2.Final.jar -jackson-annotations-2.12.7.jar -jackson-core-2.12.7.jar -jackson-databind-2.12.7.1.jar +jackson-annotations-2.12.6.jar +jackson-core-2.12.6.jar +jackson-databind-2.12.6.jar jackson-datatype-jdk8-2.13.3.jar jackson-datatype-jsr310-2.13.3.jar jackson-module-parameter-names-2.13.3.jar @@ -20,8 +19,6 @@ jakarta.annotation-api-1.3.5.jar jakarta.servlet-api-4.0.4.jar jakarta.validation-api-2.0.2.jar jakarta.websocket-api-1.1.2.jar -javax.annotation-api-1.3.2.jar -javax.el-3.0.0.jar jboss-logging-3.4.1.Final.jar jetty-continuation-9.4.46.v20220331.jar jetty-http-9.4.46.v20220331.jar @@ -37,7 +34,6 @@ jetty-xml-9.4.46.v20220331.jar jjwt-api-0.10.7.jar jjwt-impl-0.10.7.jar jjwt-jackson-0.10.7.jar -jsoup-1.14.3.jar mapstruct-1.0.0.Final.jar jsqlparser-4.4.jar mybatis-3.5.10.jar @@ -90,62 +86,53 @@ tomcat-embed-el-9.0.63.jar h2-2.1.214.jar auto-service-annotations-1.0.1.jar jsr305-3.0.0.jar -apiguardian-api-1.1.2.jar -awaitility-4.2.0.jar -checker-qual-3.10.0.jar -checkpoint-storage-api-2.3.2.jar -checkpoint-storage-local-file-2.3.2.jar +checkpoint-storage-api-2.3.3.jar +checkpoint-storage-local-file-2.3.3.jar clickhouse-cli-client-0.3.2-patch11-shaded.jar clickhouse-grpc-client-0.3.2-patch11-shaded.jar clickhouse-http-client-0.3.2-patch11-shaded.jar clickhouse-jdbc-0.3.2-patch11.jar -commons-compress-1.20.jar commons-lang3-3.4.jar -hamcrest-2.1.jar hazelcast-5.1.jar -httpclient-4.5.13.jar httpcore-4.4.13.jar httpmime-4.5.13.jar -hutool-all-5.8.16.jar jcl-over-slf4j-1.7.25.jar jcommander-1.81.jar -junit-jupiter-api-5.9.0.jar -junit-platform-commons-1.9.0.jar log4j-api-2.17.1.jar log4j-over-slf4j-1.7.25.jar log4j-to-slf4j-2.17.1.jar logback-classic-1.2.3.jar logback-core-1.2.3.jar -mssql-jdbc-9.2.1.jre8.jar -mysql-connector-java-8.0.28.jar -ojdbc8-21.5.0.0.jar -opentest4j-1.2.0.jar -postgresql-42.4.3.jar -protobuf-java-3.11.4.jar protostuff-api-1.8.0.jar protostuff-collectionschema-1.8.0.jar protostuff-core-1.8.0.jar protostuff-runtime-1.8.0.jar -redshift-jdbc42-2.1.0.10.jar -seatunnel-api-2.3.2.jar -seatunnel-common-2.3.2.jar -seatunnel-config-base-2.3.2.jar -seatunnel-config-shade-2.3.2.jar -seatunnel-core-starter-2.3.2.jar -seatunnel-engine-client-2.3.2.jar -seatunnel-engine-common-2.3.2.jar -seatunnel-engine-core-2.3.2.jar -seatunnel-guava-2.3.2-optional.jar -seatunnel-jackson-2.3.2-optional.jar -seatunnel-plugin-discovery-2.3.2.jar -seatunnel-transforms-v2-2.3.2.jar -serializer-api-2.3.2.jar -serializer-protobuf-2.3.2.jar +seatunnel-api-2.3.3.jar +seatunnel-common-2.3.3.jar +seatunnel-config-base-2.3.3.jar +seatunnel-config-shade-2.3.3.jar +seatunnel-core-starter-2.3.3.jar +seatunnel-engine-client-2.3.3.jar +seatunnel-engine-common-2.3.3.jar +seatunnel-engine-core-2.3.3.jar +seatunnel-guava-2.3.3-optional.jar +seatunnel-jackson-2.3.3-optional.jar +seatunnel-plugin-discovery-2.3.3.jar +seatunnel-transforms-v2-2.3.3.jar +serializer-api-2.3.3.jar +serializer-protobuf-2.3.3.jar swagger-annotations-2.2.14.jar -connector-cdc-mysql-2.3.2.jar -connector-jdbc-2.3.2.jar -connector-kafka-2.3.2.jar - +commons-codec-1.11.jar +commons-compress-1.20.jar +elasticsearch-rest-client-7.5.1.jar +httpasyncclient-4.1.4.jar +httpclient-4.5.13.jar +httpcore-nio-4.4.12.jar +kafka-clients-3.4.0.jar +lz4-java-1.8.0.jar +security-206.jar +snappy-java-1.1.8.4.jar +zstd-jni-1.5.2-1.jar diff --git a/tools/sonarcheck/check.sh b/tools/sonarcheck/check.sh index dc098d5b4..22b486976 100644 --- a/tools/sonarcheck/check.sh +++ b/tools/sonarcheck/check.sh @@ -18,4 +18,4 @@ if [ ! "$SONAR_TOKEN" ]; then echo "SONAR_TOKEN environment is null, skip check" exit 0 fi -./mvnw --batch-mode verify sonar:sonar -Dmaven.test.skip=true -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_incubator-seatunnel -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \ No newline at end of file +./mvnw --batch-mode verify sonar:sonar -Dmaven.test.skip=true -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_seatunnel -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \ No newline at end of file