From 02d9d136466a2b7be815f0621f2dd311b1e3e177 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 4 Mar 2024 21:06:30 -0800 Subject: [PATCH] updates --- .github/workflows/spark_sql_test.yml | 12 +++++------ dev/diffs/3.4.2.diff | 30 +++++++++++++++++----------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index f2938f5772..f78a153f27 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -45,7 +45,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-catalyst/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-catalyst/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -74,7 +74,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-core-1/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-core-1/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -103,7 +103,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-core-2/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-core-2/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -132,7 +132,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-core-3/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-core-3/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -161,7 +161,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-hive-1/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-hive-1/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -190,7 +190,7 @@ jobs: java-version: [11] spark-version: [{short: '3.4', full: '3.4.2'}] fail-fast: false - name: spark-sql-hive-2/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }} + name: spark-sql-hive-2/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust diff --git a/dev/diffs/3.4.2.diff b/dev/diffs/3.4.2.diff index 9877b9e125..454ba5408f 100644 --- a/dev/diffs/3.4.2.diff +++ b/dev/diffs/3.4.2.diff @@ -425,25 +425,19 @@ index cfc8b2cc845..c6fcfd7bd08 100644 } finally { spark.listenerManager.unregister(listener) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala -index 418ca3430bb..9524dced201 100644 +index 418ca3430bb..d5fc207601c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala -@@ -17,13 +17,10 @@ - package org.apache.spark.sql.execution - - import java.io.File -- - import scala.util.Random -- +@@ -23,7 +23,7 @@ import scala.util.Random import org.apache.hadoop.fs.Path -- + import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, QueryTest} +import org.apache.spark.sql.{DataFrame, DisableComet, QueryTest} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan import org.apache.spark.sql.internal.SQLConf -@@ -195,7 +192,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest { +@@ -195,7 +195,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest { } } @@ -780,7 +774,7 @@ index 26e61c6b58d..2a7c96d164a 100644 spark.range(10).selectExpr("id", "id % 3 as p") .write.partitionBy("p").saveAsTable("testDataForScan") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala -index 0ab8691801d..4392c74cb14 100644 +index 0ab8691801d..7deb91c40da 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala @@ -18,6 +18,7 @@ @@ -826,7 +820,7 @@ index 0ab8691801d..4392c74cb14 100644 } assert(scanNodes.length == 1) assert(scanNodes.head.output.map(_.name) == Seq("a")) -@@ -157,6 +165,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession { +@@ -157,12 +165,16 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession { val scanNodes = query.queryExecution.executedPlan.collect { case scan: BatchScanExec => scan @@ -834,6 +828,18 @@ index 0ab8691801d..4392c74cb14 100644 } assert(scanNodes.length == 1) // $"a" is not null and $"a" > 1 +- val filters = scanNodes.head.scan.asInstanceOf[ParquetScan].pushedFilters +- assert(filters.length == 2) +- assert(filters.flatMap(_.references).distinct === Array("a")) ++ val dataFilters = scanNodes.head match { ++ case scan: FileSourceScanExec => scan.dataFilters ++ case scan: CometScanExec => scan.dataFilters ++ } ++ assert(dataFilters.length == 2) ++ assert(dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a")) + } + } + } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala index 266bb343526..85ec36db996 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala