Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[EnginePlugin][Spark]Turn off use secure random by default #5197

Merged
merged 3 commits into from
Nov 12, 2024
Merged

[EnginePlugin][Spark]Turn off use secure random by default #5197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7a7ecc
Turn off use secure random by default close #5196
peacewong Nov 6, 2024
2666925
Update Notification Mailing List
peacewong Nov 11, 2024
d10ee78
Fix ds meta service build
peacewong Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .asf.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,6 @@ github:
required_approving_review_count: 1
notifications:
commits: [email protected]
issues: notifications@linkis.apache.org
pullrequests: notifications@linkis.apache.org
issues: dev@linkis.apache.org
pullrequests: dev@linkis.apache.org

5 changes: 4 additions & 1 deletion ...spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ object SparkConfiguration extends Logging {
CommonVars("wds.linkis.spark.engineconn.fatal.log", "error writing class;OutOfMemoryError")

val PYSPARK_PYTHON3_PATH =
CommonVars[String]("pyspark.python3.path", "/appcom/Install/anaconda3/bin/python")
CommonVars[String]("pyspark.python3.path", "python3")

val ENABLE_REPLACE_PACKAGE_NAME =
CommonVars("wds.linkis.spark.engine.scala.replace_package_header.enable", true)
Expand All @@ -182,6 +182,9 @@ object SparkConfiguration extends Logging {

val LINKIS_SPARK_ETL_SUPPORT_HUDI = CommonVars[Boolean]("linkis.spark.etl.support.hudi", false)

val LINKIS_PYSPARK_USE_SECURE_RANDOM =
CommonVars[Boolean]("linkis.pyspark.use.secure.random", false).getValue

val SCALA_PARSE_APPEND_CODE =
CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue

Expand Down
9 changes: 7 additions & 2 deletions ...rk/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import org.apache.spark.sql.execution.datasources.csv.UDF

import java.io._
import java.net.InetAddress
import java.security.SecureRandom
import java.util

import scala.collection.JavaConverters._
Expand Down Expand Up @@ -76,7 +77,12 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In
private val lineOutputStream = new RsOutputStream
val sqlContext = sparkEngineSession.sqlContext
val SUCCESS = "success"
private lazy val py4jToken: String = SecureRandomStringUtils.randomAlphanumeric(256)

private lazy val py4jToken: String = if (SparkConfiguration.LINKIS_PYSPARK_USE_SECURE_RANDOM) {
SecureRandomStringUtils.randomAlphanumeric(256)
} else {
SecureRandom.getInstance("SHA1PRNG").nextInt(100000).toString
}

private lazy val gwBuilder: GatewayServerBuilder = {
val builder = new GatewayServerBuilder()
Expand Down Expand Up @@ -152,7 +158,6 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In
)
val userDefinePythonVersion = engineCreationContext.getOptions
.getOrDefault("spark.python.version", "python")
.toString
.toLowerCase()
val sparkPythonVersion =
if (
Expand Down
2 changes: 1 addition & 1 deletion linkis-public-enhancements/distribution.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@
<fileSets>
<fileSet>
<directory>
./linkis-datasource/linkis-datasource-manager/server/target/out/lib/service
./linkis-datasource/linkis-datasource-manager/server/target/out/service
</directory>
<outputDirectory>
lib/metadataquery-service
Expand Down
12 changes: 6 additions & 6 deletions ...nts/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@
<fileSet>
<directory>${basedir}/../service/elasticsearch/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/elasticsearch</outputDirectory>
<outputDirectory>service/elasticsearch</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand All @@ -198,7 +198,7 @@
<fileSet>
<directory>${basedir}/../service/hive/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/hive</outputDirectory>
<outputDirectory>service/hive</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand All @@ -209,7 +209,7 @@
<fileSet>
<directory>${basedir}/../service/kafka/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/kafka</outputDirectory>
<outputDirectory>service/kafka</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand All @@ -220,7 +220,7 @@
<fileSet>
<directory>${basedir}/../service/jdbc/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/jdbc</outputDirectory>
<outputDirectory>service/jdbc</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand All @@ -231,7 +231,7 @@
<fileSet>
<directory>${basedir}/../service/hdfs/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/hdfs</outputDirectory>
<outputDirectory>service/hdfs</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand All @@ -242,7 +242,7 @@
<fileSet>
<directory>${basedir}/../service/mongodb/target/out/lib</directory>
<fileMode>0755</fileMode>
<outputDirectory>lib/service/mongodb</outputDirectory>
<outputDirectory>service/mongodb</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
Expand Down
Loading