Upgrade spark and dgraph versions (#265)

## Checklist before submitting - [x] Did you read the [contributing guide](https://github.com/G-Research/spark-dgraph-connector/blob/contributing-guidelines/CONTRIBUTING.md)? - [ ] Did you update the docs? - [ ] Did you write any tests to validate this change? ## Description Upgrades Spark and Dgraph versions tested in CI. ## Review process for approval 1. All tests and other checks must succeed. 2. At least one core contributors must review and approve. 3. If a core contributor requests changes, they must be addressed.
G-Research · Jun 17, 2024 · 9a86cc7 · 9a86cc7
1 parent 09dacbf
commit 9a86cc7
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 93 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -29,11 +29,11 @@ jobs:
             scala-compat-version: '2.12'
             scala-version: '2.12.15'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.0'
+            spark-version: '3.5.1'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
 
@@ -46,11 +46,11 @@ jobs:
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.3'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.0'
+            spark-version: '3.5.1'
             scala-compat-version: '2.13'
             scala-version: '2.13.8'
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -38,8 +38,8 @@ jobs:
         include:
           - dgraph-version: "21.12.0"
           - dgraph-version: "22.0.2"
-          - dgraph-version: "23.0.1"
-          - dgraph-version: "23.1.0"
+          - dgraph-version: "23.1.1"
+          - dgraph-version: "24.0.0"
     uses: "./.github/workflows/test-spark.yml"
     with:
       dgraph-version: ${{ matrix.dgraph-version }}
@@ -52,103 +52,33 @@ jobs:
       matrix: |
         {
           "include": [
-            {
-              "spark-version": "3.2.0",
-              "spark-compat-version": "3.2",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.5",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.2.1",
-              "spark-compat-version": "3.2",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.5",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.2.2",
-              "spark-compat-version": "3.2",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.5",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.2.3",
-              "spark-compat-version": "3.2",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.5",
-              "dgraph-version": "22.0.2"
-            },
             {
               "spark-version": "3.2.4",
               "spark-compat-version": "3.2",
               "scala-compat-version": "2.13",
               "scala-version": "2.13.5",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.3.0",
-              "spark-compat-version": "3.3",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.3.1",
-              "spark-compat-version": "3.3",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.3.2",
-              "spark-compat-version": "3.3",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.3.3",
-              "spark-compat-version": "3.3",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
+              "dgraph-version": "23.1.1"
             },
             {
               "spark-version": "3.3.4",
               "spark-compat-version": "3.3",
               "scala-compat-version": "2.13",
               "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.4.0",
-              "spark-compat-version": "3.4",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
-            },
-            {
-              "spark-version": "3.4.1",
-              "spark-compat-version": "3.4",
-              "scala-compat-version": "2.13",
-              "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
+              "dgraph-version": "23.1.1"
             },
             {
-              "spark-version": "3.4.2",
+              "spark-version": "3.4.3",
               "spark-compat-version": "3.4",
               "scala-compat-version": "2.13",
               "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
+              "dgraph-version": "23.1.1"
             },
             {
-              "spark-version": "3.5.0",
+              "spark-version": "3.5.1",
               "spark-compat-version": "3.5",
               "scala-compat-version": "2.13",
               "scala-version": "2.13.8",
-              "dgraph-version": "22.0.2"
+              "dgraph-version": "23.1.1"
             }
           ]
         }

diff --git a/.github/workflows/test-dgraph.yml b/.github/workflows/test-dgraph.yml
@@ -19,12 +19,13 @@ jobs:
             "21.03.0", "21.03.1", "21.03.2", "21.12.0",
             "22.0.0", "22.0.1", "22.0.2",
             "23.0.0", "23.0.1",
-            "23.1.0"
+            "23.1.0", "23.1.1",
+            "24.0.0"
         ]
         scala-compat-version: ["2.12"]
         include:
           - spark-compat-version: '3.5'
-            spark-version: '3.5.0'
+            spark-version: '3.5.1'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
 

diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         spark-compat-version: ["3.2", "3.3", "3.4", "3.5"]
-        dgraph-minor-version: ["20.03", "20.07", "20.11", "21.12", "22.0", "23.0", "23.1"]
+        dgraph-minor-version: ["20.03", "20.07", "20.11", "21.12", "22.0", "23.0", "23.1", "24.0"]
         scala-compat-version: ["2.12"]
         include:
           - spark-compat-version: '3.0'
@@ -42,12 +42,12 @@ jobs:
             scala-version: '2.12.15'
             hadoop-version: '3'
           - spark-compat-version: '3.4'
-            spark-version: '3.4.2'
+            spark-version: '3.4.3'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             hadoop-version: '3'
           - spark-compat-version: '3.5'
-            spark-version: '3.5.0'
+            spark-version: '3.5.1'
             scala-compat-version: '2.12'
             scala-version: '2.12.17'
             hadoop-version: '3'
@@ -65,7 +65,9 @@ jobs:
           - dgraph-minor-version: "23.0"
             dgraph-version: "23.0.1"
           - dgraph-minor-version: "23.1"
-            dgraph-version: "23.1.0"
+            dgraph-version: "23.1.1"
+          - dgraph-minor-version: "24.0"
+            dgraph-version: "24.0.0"
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -13,11 +13,11 @@ jobs:
       matrix:
         spark-compat-version: ["3.5"]
         scala-compat-version: ["2.12"]
-        dgraph-minor-version: ["21.12", "22.0", "23.0", "23.1"]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3"]
+        dgraph-minor-version: ["21.12", "22.0", "23.0", "23.1", "24.0"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         include:
           - spark-compat-version: '3.5'
-            spark-version: '3.5.0'
+            spark-version: '3.5.1'
             scala-compat-version: '2.12'
             scala-version: '2.12.18'
 
@@ -28,7 +28,9 @@ jobs:
           - dgraph-minor-version: "23.0"
             dgraph-version: "23.0.1"
           - dgraph-minor-version: "23.1"
-            dgraph-version: "23.1.0"
+            dgraph-version: "23.1.1"
+          - dgraph-minor-version: "24.0"
+            dgraph-version: "24.0.0"
 
     steps:
       - name: Checkout

diff --git a/pom.xml b/pom.xml
@@ -40,7 +40,7 @@
     <scala.version>${scala.compat.version}.18</scala.version>
     <!-- spark.compat.version must be reflected in project version -->
     <spark.compat.version>3.5</spark.compat.version>
-    <spark.version>${spark.compat.version}.0</spark.version>
+    <spark.version>${spark.compat.version}.1</spark.version>
     <graphframes.version>0.8.3-spark3.5</graphframes.version>
   </properties>