Update pyspark versions (#835)

Signed-off-by: GBBBAS <bryce.bartmann@shell.com>
rtdip · Oct 29, 2024 · b3f674e · b3f674e
1 parent 975f96c
commit b3f674e
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 8 deletions.
diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml
@@ -48,8 +48,8 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.11"]
-        pyspark: ["3.5.2"]
-        delta-spark: ["3.2.1"]
+        pyspark: ["3.5.3"]
+        delta-spark: ["3.0.0"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -27,16 +27,30 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11"]
-        pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"]
+        pyspark:
+          [
+            "3.3.0",
+            "3.3.1",
+            "3.3.2",
+            "3.4.0",
+            "3.4.1",
+            "3.5.0",
+            "3.5.1",
+            "3.5.2",
+          ]
         exclude:
-          - pyspark: "3.5.0"
+          - pyspark: "3.5.2"
             python-version: "3.9"
-          - pyspark: "3.5.0"
+          - pyspark: "3.5.2"
             python-version: "3.10"
           - pyspark: "3.5.1"
             python-version: "3.9"
           - pyspark: "3.5.1"
             python-version: "3.10"
+          - pyspark: "3.5.0"
+            python-version: "3.9"
+          - pyspark: "3.5.0"
+            python-version: "3.10"
           - pyspark: "3.4.1"
             python-version: "3.9"
           - pyspark: "3.4.1"
@@ -65,7 +79,9 @@ jobs:
           - pyspark: "3.5.0"
             delta-spark: "3.0.0"
           - pyspark: "3.5.1"
-            delta-spark: "3.1.0"
+            delta-spark: "3.0.0"
+          - pyspark: "3.5.2"
+            delta-spark: "3.0.0"
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -109,8 +125,8 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.11"]
-        pyspark: ["3.5.2"]
-        delta-spark: ["3.2.1"]
+        pyspark: ["3.5.3"]
+        delta-spark: ["3.0.0"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4