chore: upgrade Spark to 3.4, and Deequ to 2.0.5

awslabs · Oct 25, 2023 · 1004a9d · 1004a9d
1 parent e74e974
commit 1004a9d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        PYSPARK_VERSION: ["3.0", "3.1.3", "3.2", "3.3"]
+        PYSPARK_VERSION: ["3.0", "3.1.3", "3.2", "3.3", "3.4"]
 
     steps:
       - uses: actions/checkout@v3

diff --git a/pydeequ/configs.py b/pydeequ/configs.py
@@ -5,6 +5,7 @@
 
 
 SPARK_TO_DEEQU_COORD_MAPPING = {
+    "3.4": "com.amazon.deequ:deequ:2.0.5-spark-3.4",
     "3.3": "com.amazon.deequ:deequ:2.0.3-spark-3.3",
     "3.2": "com.amazon.deequ:deequ:2.0.1-spark-3.2",
     "3.1": "com.amazon.deequ:deequ:2.0.0-spark-3.1",