updates

sunchao · Mar 5, 2024 · 02d9d13 · 02d9d13
1 parent 194a5e4
commit 02d9d13
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 18 deletions.
diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml
@@ -45,7 +45,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-catalyst/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-catalyst/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -74,7 +74,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-core-1/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-core-1/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -103,7 +103,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-core-2/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-core-2/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -132,7 +132,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-core-3/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-core-3/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -161,7 +161,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-hive-1/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-hive-1/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -190,7 +190,7 @@ jobs:
         java-version: [11]
         spark-version: [{short: '3.4', full: '3.4.2'}]
       fail-fast: false
-    name: spark-sql-hive-2/${{ matrix.os }}/spark-${{ matrix.spark-version }}/java-${{ matrix.java-version }}
+    name: spark-sql-hive-2/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust

diff --git a/dev/diffs/3.4.2.diff b/dev/diffs/3.4.2.diff
@@ -425,25 +425,19 @@ index cfc8b2cc845..c6fcfd7bd08 100644
          } finally {
            spark.listenerManager.unregister(listener)
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-index 418ca3430bb..9524dced201 100644
+index 418ca3430bb..d5fc207601c 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DataSourceScanExecRedactionSuite.scala
-@@ -17,13 +17,10 @@
- package org.apache.spark.sql.execution
-
- import java.io.File
--
- import scala.util.Random
--
+@@ -23,7 +23,7 @@ import scala.util.Random
  import org.apache.hadoop.fs.Path
--
+ 
  import org.apache.spark.SparkConf
 -import org.apache.spark.sql.{DataFrame, QueryTest}
 +import org.apache.spark.sql.{DataFrame, DisableComet, QueryTest}
  import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
  import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan
  import org.apache.spark.sql.internal.SQLConf
-@@ -195,7 +192,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest {
+@@ -195,7 +195,7 @@ class DataSourceV2ScanExecRedactionSuite extends DataSourceScanRedactionTest {
      }
    }
 
@@ -780,7 +774,7 @@ index 26e61c6b58d..2a7c96d164a 100644
        spark.range(10).selectExpr("id", "id % 3 as p")
          .write.partitionBy("p").saveAsTable("testDataForScan")
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
-index 0ab8691801d..4392c74cb14 100644
+index 0ab8691801d..7deb91c40da 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFsSuite.scala
 @@ -18,6 +18,7 @@
@@ -826,14 +820,26 @@ index 0ab8691801d..4392c74cb14 100644
            }
            assert(scanNodes.length == 1)
            assert(scanNodes.head.output.map(_.name) == Seq("a"))
-@@ -157,6 +165,7 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
+@@ -157,12 +165,16 @@ class ExtractPythonUDFsSuite extends SparkPlanTest with SharedSparkSession {
 
            val scanNodes = query.queryExecution.executedPlan.collect {
              case scan: BatchScanExec => scan
 +            case scan: CometScanExec => scan
            }
            assert(scanNodes.length == 1)
            // $"a" is not null and $"a" > 1
+-          val filters = scanNodes.head.scan.asInstanceOf[ParquetScan].pushedFilters
+-          assert(filters.length == 2)
+-          assert(filters.flatMap(_.references).distinct === Array("a"))
++          val dataFilters = scanNodes.head match {
++            case scan: FileSourceScanExec => scan.dataFilters
++            case scan: CometScanExec => scan.dataFilters
++          }
++          assert(dataFilters.length == 2)
++          assert(dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a"))
+         }
+       }
+     }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
 index 266bb343526..85ec36db996 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala