diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index d5f5fd2e2150..e52c53d1d558 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -65,7 +65,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { postBuildInfoEvent(sc) - setPredefinedConfigs(sc, conf) + setPredefinedConfigs(conf) // Initialize Backend. Backend.get().onDriverStart(sc, pluginContext) @@ -123,7 +123,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { GlutenEventUtils.post(sc, event) } - private def setPredefinedConfigs(sc: SparkContext, conf: SparkConf): Unit = { + private def setPredefinedConfigs(conf: SparkConf): Unit = { // Spark SQL extensions val extensions = if (conf.contains(SPARK_SESSION_EXTENSIONS.key)) { s"${conf.get(SPARK_SESSION_EXTENSIONS.key)}," + @@ -134,7 +134,10 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { conf.set(SPARK_SESSION_EXTENSIONS.key, extensions) // adaptive custom cost evaluator class - if (GlutenConfig.getConf.enableGluten && GlutenConfig.getConf.enableGlutenCostEvaluator) { + val enableGlutenCostEvaluator = conf.getBoolean( + GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED, + GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE) + if (enableGlutenCostEvaluator) { val costEvaluator = "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator" conf.set(SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key, costEvaluator) } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 0bfe16270e5f..0e41fc65376c 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -466,8 +466,6 @@ class GlutenConfig(conf: SQLConf) extends Logging { def enableCastAvgAggregateFunction: Boolean = conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) - def enableGlutenCostEvaluator: Boolean = conf.getConf(COST_EVALUATOR_ENABLED) - def dynamicOffHeapSizingEnabled: Boolean = conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) @@ -648,6 +646,7 @@ object GlutenConfig { "spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction" val GLUTEN_COST_EVALUATOR_ENABLED = "spark.gluten.sql.adaptive.costEvaluator.enabled" + val GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE = true var ins: GlutenConfig = _ @@ -2116,15 +2115,15 @@ object GlutenConfig { .createWithDefault(true) val COST_EVALUATOR_ENABLED = - buildConf(GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED) + buildStaticConf(GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED) .internal() .doc( - "If true and gluten enabled, use " + + "If true, use " + "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost " + "evaluator class, else follow the configuration " + "spark.sql.adaptive.customCostEvaluatorClass.") .booleanConf - .createWithDefault(true) + .createWithDefault(GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE) val DYNAMIC_OFFHEAP_SIZING_ENABLED = buildConf(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED)