From 9a86cc7ed86c9bf59f2c42bd7d963d726040a81e Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Mon, 17 Jun 2024 10:39:15 +0200 Subject: [PATCH] Upgrade spark and dgraph versions (#265) ## Checklist before submitting - [x] Did you read the [contributing guide](https://github.com/G-Research/spark-dgraph-connector/blob/contributing-guidelines/CONTRIBUTING.md)? - [ ] Did you update the docs? - [ ] Did you write any tests to validate this change? ## Description Upgrades Spark and Dgraph versions tested in CI. ## Review process for approval 1. All tests and other checks must succeed. 2. At least one core contributors must review and approve. 3. If a core contributor requests changes, they must be addressed. --- .github/workflows/build.yml | 8 +-- .github/workflows/ci.yml | 86 +++----------------------- .github/workflows/test-dgraph.yml | 5 +- .github/workflows/test-integration.yml | 10 +-- .github/workflows/test-python.yml | 10 +-- pom.xml | 2 +- 6 files changed, 28 insertions(+), 93 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59209179..15f83548 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,11 +29,11 @@ jobs: scala-compat-version: '2.12' scala-version: '2.12.15' - spark-compat-version: '3.4' - spark-version: '3.4.2' + spark-version: '3.4.3' scala-compat-version: '2.12' scala-version: '2.12.17' - spark-compat-version: '3.5' - spark-version: '3.5.0' + spark-version: '3.5.1' scala-compat-version: '2.12' scala-version: '2.12.18' @@ -46,11 +46,11 @@ jobs: scala-compat-version: '2.13' scala-version: '2.13.8' - spark-compat-version: '3.4' - spark-version: '3.4.2' + spark-version: '3.4.3' scala-compat-version: '2.13' scala-version: '2.13.8' - spark-compat-version: '3.5' - spark-version: '3.5.0' + spark-version: '3.5.1' scala-compat-version: '2.13' scala-version: '2.13.8' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6f7ba28..b886e546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,8 @@ jobs: include: - dgraph-version: "21.12.0" - dgraph-version: "22.0.2" - - dgraph-version: "23.0.1" - - dgraph-version: "23.1.0" + - dgraph-version: "23.1.1" + - dgraph-version: "24.0.0" uses: "./.github/workflows/test-spark.yml" with: dgraph-version: ${{ matrix.dgraph-version }} @@ -52,103 +52,33 @@ jobs: matrix: | { "include": [ - { - "spark-version": "3.2.0", - "spark-compat-version": "3.2", - "scala-compat-version": "2.13", - "scala-version": "2.13.5", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.2.1", - "spark-compat-version": "3.2", - "scala-compat-version": "2.13", - "scala-version": "2.13.5", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.2.2", - "spark-compat-version": "3.2", - "scala-compat-version": "2.13", - "scala-version": "2.13.5", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.2.3", - "spark-compat-version": "3.2", - "scala-compat-version": "2.13", - "scala-version": "2.13.5", - "dgraph-version": "22.0.2" - }, { "spark-version": "3.2.4", "spark-compat-version": "3.2", "scala-compat-version": "2.13", "scala-version": "2.13.5", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.3.0", - "spark-compat-version": "3.3", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.3.1", - "spark-compat-version": "3.3", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.3.2", - "spark-compat-version": "3.3", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.3.3", - "spark-compat-version": "3.3", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" + "dgraph-version": "23.1.1" }, { "spark-version": "3.3.4", "spark-compat-version": "3.3", "scala-compat-version": "2.13", "scala-version": "2.13.8", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.4.0", - "spark-compat-version": "3.4", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" - }, - { - "spark-version": "3.4.1", - "spark-compat-version": "3.4", - "scala-compat-version": "2.13", - "scala-version": "2.13.8", - "dgraph-version": "22.0.2" + "dgraph-version": "23.1.1" }, { - "spark-version": "3.4.2", + "spark-version": "3.4.3", "spark-compat-version": "3.4", "scala-compat-version": "2.13", "scala-version": "2.13.8", - "dgraph-version": "22.0.2" + "dgraph-version": "23.1.1" }, { - "spark-version": "3.5.0", + "spark-version": "3.5.1", "spark-compat-version": "3.5", "scala-compat-version": "2.13", "scala-version": "2.13.8", - "dgraph-version": "22.0.2" + "dgraph-version": "23.1.1" } ] } diff --git a/.github/workflows/test-dgraph.yml b/.github/workflows/test-dgraph.yml index 97afc3be..33fafbf0 100644 --- a/.github/workflows/test-dgraph.yml +++ b/.github/workflows/test-dgraph.yml @@ -19,12 +19,13 @@ jobs: "21.03.0", "21.03.1", "21.03.2", "21.12.0", "22.0.0", "22.0.1", "22.0.2", "23.0.0", "23.0.1", - "23.1.0" + "23.1.0", "23.1.1", + "24.0.0" ] scala-compat-version: ["2.12"] include: - spark-compat-version: '3.5' - spark-version: '3.5.0' + spark-version: '3.5.1' scala-compat-version: '2.12' scala-version: '2.12.18' diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 56e3dfbd..88ff100e 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: spark-compat-version: ["3.2", "3.3", "3.4", "3.5"] - dgraph-minor-version: ["20.03", "20.07", "20.11", "21.12", "22.0", "23.0", "23.1"] + dgraph-minor-version: ["20.03", "20.07", "20.11", "21.12", "22.0", "23.0", "23.1", "24.0"] scala-compat-version: ["2.12"] include: - spark-compat-version: '3.0' @@ -42,12 +42,12 @@ jobs: scala-version: '2.12.15' hadoop-version: '3' - spark-compat-version: '3.4' - spark-version: '3.4.2' + spark-version: '3.4.3' scala-compat-version: '2.12' scala-version: '2.12.17' hadoop-version: '3' - spark-compat-version: '3.5' - spark-version: '3.5.0' + spark-version: '3.5.1' scala-compat-version: '2.12' scala-version: '2.12.17' hadoop-version: '3' @@ -65,7 +65,9 @@ jobs: - dgraph-minor-version: "23.0" dgraph-version: "23.0.1" - dgraph-minor-version: "23.1" - dgraph-version: "23.1.0" + dgraph-version: "23.1.1" + - dgraph-minor-version: "24.0" + dgraph-version: "24.0.0" steps: - name: Checkout diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 14243814..8feb4f73 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -13,11 +13,11 @@ jobs: matrix: spark-compat-version: ["3.5"] scala-compat-version: ["2.12"] - dgraph-minor-version: ["21.12", "22.0", "23.0", "23.1"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3"] + dgraph-minor-version: ["21.12", "22.0", "23.0", "23.1", "24.0"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - spark-compat-version: '3.5' - spark-version: '3.5.0' + spark-version: '3.5.1' scala-compat-version: '2.12' scala-version: '2.12.18' @@ -28,7 +28,9 @@ jobs: - dgraph-minor-version: "23.0" dgraph-version: "23.0.1" - dgraph-minor-version: "23.1" - dgraph-version: "23.1.0" + dgraph-version: "23.1.1" + - dgraph-minor-version: "24.0" + dgraph-version: "24.0.0" steps: - name: Checkout diff --git a/pom.xml b/pom.xml index 16a0903d..8d04d0cb 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ ${scala.compat.version}.18 3.5 - ${spark.compat.version}.0 + ${spark.compat.version}.1 0.8.3-spark3.5