From 1004a9da1ca2425a9b0471cd814a079f40b3e33c Mon Sep 17 00:00:00 2001 From: Chenyang Liu Date: Wed, 25 Oct 2023 16:40:53 -0400 Subject: [PATCH] chore: upgrade Spark to 3.4, and Deequ to 2.0.5 --- .github/workflows/base.yml | 2 +- pydeequ/configs.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 1bce4a0..2c7acc6 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - PYSPARK_VERSION: ["3.0", "3.1.3", "3.2", "3.3"] + PYSPARK_VERSION: ["3.0", "3.1.3", "3.2", "3.3", "3.4"] steps: - uses: actions/checkout@v3 diff --git a/pydeequ/configs.py b/pydeequ/configs.py index c3c885d..c101872 100644 --- a/pydeequ/configs.py +++ b/pydeequ/configs.py @@ -5,6 +5,7 @@ SPARK_TO_DEEQU_COORD_MAPPING = { + "3.4": "com.amazon.deequ:deequ:2.0.5-spark-3.4", "3.3": "com.amazon.deequ:deequ:2.0.3-spark-3.3", "3.2": "com.amazon.deequ:deequ:2.0.1-spark-3.2", "3.1": "com.amazon.deequ:deequ:2.0.0-spark-3.1",