diff --git a/CHANGELOG.md b/CHANGELOG.md index 48615f3db..1cad322f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ All notable changes to this project will be documented in this file. - yq: Bump products to use `4.45.2` ([#1090]). - cyclonedx-bom: Bump airflow and superset to use `6.0.0` ([#1090]). - vector: Bump to `0.46.1` ([#1098]). +- spark: update dependencies for 3.5.5 ([#1094]) ### Fixed @@ -80,6 +81,7 @@ All notable changes to this project will be documented in this file. Also remove the old release workflow. - zookeeper: Remove 3.9.2 ([#1093]). - Remove ubi8-rust-builder image ([#1091]). +- spark: remove 3.5.2 ([#1094]) - hadoop: Remove `3.3.4` and `3.4.0` ([#1099]). - opa: Remove `0.67.1` ([#1103]). - opa: Remove legacy bundle-builder from container build ([#1103]). @@ -110,6 +112,7 @@ All notable changes to this project will be documented in this file. [#1090]: https://github.com/stackabletech/docker-images/pull/1090 [#1091]: https://github.com/stackabletech/docker-images/pull/1091 [#1093]: https://github.com/stackabletech/docker-images/pull/1093 +[#1094]: https://github.com/stackabletech/docker-images/pull/1094 [#1097]: https://github.com/stackabletech/docker-images/pull/1097 [#1098]: https://github.com/stackabletech/docker-images/pull/1098 [#1099]: https://github.com/stackabletech/docker-images/pull/1099 diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 05e39c9a1..88b9dce78 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -155,7 +155,7 @@ WORKDIR /stackable/spark-${PRODUCT}/dist/jars # Copy modules required for s3a:// COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \ /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar \ - /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE}.jar \ ./ # Copy modules required for abfs:// diff --git a/spark-k8s/stackable/patches/3.5.2/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.2/0001-Update-CycloneDX-plugin.patch deleted file mode 100644 index 079885f0b..000000000 --- a/spark-k8s/stackable/patches/3.5.2/0001-Update-CycloneDX-plugin.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 2f95ba96e5894cfd07eca25aef5968e6a6d543fd Mon Sep 17 00:00:00 2001 -From: Lukas Voetmand -Date: Fri, 6 Sep 2024 17:53:52 +0200 -Subject: Update CycloneDX plugin - ---- - dev/make-distribution.sh | 1 - - pom.xml | 7 ++++++- - 2 files changed, 6 insertions(+), 2 deletions(-) - -diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh -index ef7c010e930..0f4c1c74e40 100755 ---- a/dev/make-distribution.sh -+++ b/dev/make-distribution.sh -@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ - -Dmaven.javadoc.skip=true \ - -Dmaven.scaladoc.skip=true \ - -Dmaven.source.skip \ -- -Dcyclonedx.skip=true \ - $@) - - # Actually build the jar -diff --git a/pom.xml b/pom.xml -index 8fe98c35846..36a800162da 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3513,7 +3513,12 @@ - - org.cyclonedx - cyclonedx-maven-plugin -- 2.7.9 -+ 2.8.0 -+ -+ application -+ 1.5 -+ false -+ - - - package diff --git a/spark-k8s/stackable/patches/3.5.2/patchable.toml b/spark-k8s/stackable/patches/3.5.2/patchable.toml deleted file mode 100644 index 98aa7ed62..000000000 --- a/spark-k8s/stackable/patches/3.5.2/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -base = "bb7846dd487f259994fdc69e18e03382e3f64f42" -mirror = "https://github.com/stackabletech/spark.git" diff --git a/spark-k8s/versions.py b/spark-k8s/versions.py index 0fedd8b23..851587858 100644 --- a/spark-k8s/versions.py +++ b/spark-k8s/versions.py @@ -1,31 +1,13 @@ versions = [ - { - "product": "3.5.2", - "java-base": "17", - "java-devel": "17", - "python": "3.11", - "hadoop": "3.3.6", # Hadoop version defined in ../hbase/versions.py to reduce build time and disk requirements - "hbase": "2.6.1", # current Stackable LTS version - "aws_java_sdk_bundle": "1.12.367", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6 - "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "vector": "0.46.1", - "jmx_exporter": "1.2.0", - "tini": "0.19.0", - "hbase_connector": "1.0.1", - }, { "product": "3.5.5", "java-base": "17", "java-devel": "17", "python": "3.11", - "hadoop": "3.3.6", # Hadoop version defined in ../hbase/versions.py to reduce build time and disk requirements - "hbase": "2.6.1", # current Stackable LTS version - "aws_java_sdk_bundle": "1.12.367", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6 + "hadoop": "3.4.1", # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. + "hbase": "2.6.2", # Current Stackable LTS version. Used to build the HBase connector. + "aws_java_sdk_bundle": "2.24.6", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2