From a7a7ecc2d7716e4393e3baa51ac9915c2b078b00 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 6 Nov 2024 20:02:41 +0800 Subject: [PATCH 1/3] Turn off use secure random by default close #5196 --- .../engineplugin/spark/config/SparkConfiguration.scala | 5 ++++- .../spark/executor/SparkPythonExecutor.scala | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala index a493c5ff37..429048c77f 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala @@ -156,7 +156,7 @@ object SparkConfiguration extends Logging { CommonVars("wds.linkis.spark.engineconn.fatal.log", "error writing class;OutOfMemoryError") val PYSPARK_PYTHON3_PATH = - CommonVars[String]("pyspark.python3.path", "/appcom/Install/anaconda3/bin/python") + CommonVars[String]("pyspark.python3.path", "python3") val ENABLE_REPLACE_PACKAGE_NAME = CommonVars("wds.linkis.spark.engine.scala.replace_package_header.enable", true) @@ -182,6 +182,9 @@ object SparkConfiguration extends Logging { val LINKIS_SPARK_ETL_SUPPORT_HUDI = CommonVars[Boolean]("linkis.spark.etl.support.hudi", false) + val LINKIS_PYSPARK_USE_SECURE_RANDOM = + CommonVars[Boolean]("linkis.pyspark.use.secure.random", false).getValue + val SCALA_PARSE_APPEND_CODE = CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala index 5d4305c67c..f947db9338 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala @@ -46,6 +46,7 @@ import org.apache.spark.sql.execution.datasources.csv.UDF import java.io._ import java.net.InetAddress +import java.security.SecureRandom import java.util import scala.collection.JavaConverters._ @@ -76,7 +77,12 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In private val lineOutputStream = new RsOutputStream val sqlContext = sparkEngineSession.sqlContext val SUCCESS = "success" - private lazy val py4jToken: String = SecureRandomStringUtils.randomAlphanumeric(256) + + private lazy val py4jToken: String = if (SparkConfiguration.LINKIS_PYSPARK_USE_SECURE_RANDOM) { + SecureRandomStringUtils.randomAlphanumeric(256) + } else { + SecureRandom.getInstance("SHA1PRNG").nextInt(100000).toString + } private lazy val gwBuilder: GatewayServerBuilder = { val builder = new GatewayServerBuilder() @@ -152,7 +158,6 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In ) val userDefinePythonVersion = engineCreationContext.getOptions .getOrDefault("spark.python.version", "python") - .toString .toLowerCase() val sparkPythonVersion = if ( From 26669254fa4ff53084c7c95b2cc831830f1b6d44 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 11 Nov 2024 15:55:21 +0800 Subject: [PATCH 2/3] Update Notification Mailing List --- .asf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 79396eba55..08ec821f75 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -67,6 +67,6 @@ github: required_approving_review_count: 1 notifications: commits: commits@linkis.apache.org - issues: notifications@linkis.apache.org - pullrequests: notifications@linkis.apache.org + issues: dev@linkis.apache.org + pullrequests: dev@linkis.apache.org From d10ee784ed57db15a28ae3b6c420a10473aa95a5 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 11 Nov 2024 20:38:22 +0800 Subject: [PATCH 3/3] Fix ds meta service build --- linkis-public-enhancements/distribution.xml | 2 +- .../server/src/main/assembly/distribution.xml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/linkis-public-enhancements/distribution.xml b/linkis-public-enhancements/distribution.xml index 3e3baf6396..b4cc88582e 100644 --- a/linkis-public-enhancements/distribution.xml +++ b/linkis-public-enhancements/distribution.xml @@ -290,7 +290,7 @@ - ./linkis-datasource/linkis-datasource-manager/server/target/out/lib/service + ./linkis-datasource/linkis-datasource-manager/server/target/out/service lib/metadataquery-service diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml index 26130980b3..e124c84fc4 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml @@ -187,7 +187,7 @@ ${basedir}/../service/elasticsearch/target/out/lib 0755 - lib/service/elasticsearch + service/elasticsearch *.jar @@ -198,7 +198,7 @@ ${basedir}/../service/hive/target/out/lib 0755 - lib/service/hive + service/hive *.jar @@ -209,7 +209,7 @@ ${basedir}/../service/kafka/target/out/lib 0755 - lib/service/kafka + service/kafka *.jar @@ -220,7 +220,7 @@ ${basedir}/../service/jdbc/target/out/lib 0755 - lib/service/jdbc + service/jdbc *.jar @@ -231,7 +231,7 @@ ${basedir}/../service/hdfs/target/out/lib 0755 - lib/service/hdfs + service/hdfs *.jar @@ -242,7 +242,7 @@ ${basedir}/../service/mongodb/target/out/lib 0755 - lib/service/mongodb + service/mongodb *.jar