diff --git a/CHANGELOG.md b/CHANGELOG.md index e827a8c6..0137699a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. ### Changed - Default to OCI for image metadata and product image selection ([#514]). +- Update tests and docs to Spark version 3.5.5 ([#534]) [#501]: https://github.com/stackabletech/spark-k8s-operator/pull/501 [#508]: https://github.com/stackabletech/spark-k8s-operator/pull/508 @@ -23,6 +24,7 @@ All notable changes to this project will be documented in this file. [#515]: https://github.com/stackabletech/spark-k8s-operator/pull/515 [#528]: https://github.com/stackabletech/spark-k8s-operator/pull/528 [#532]: https://github.com/stackabletech/spark-k8s-operator/pull/532 +[#534]: https://github.com/stackabletech/spark-k8s-operator/pull/534 ## [24.11.1] - 2025-01-10 diff --git a/Cargo.lock b/Cargo.lock index 04b5971c..3f1ae14a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.31" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" dependencies = [ "clap_builder", "clap_derive", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.31" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" dependencies = [ "anstream", "anstyle", @@ -323,9 +323,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck", "proc-macro2", @@ -896,9 +896,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -917,12 +917,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", + "futures-core", "http", "http-body", "pin-project-lite", @@ -1200,9 +1200,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1620,9 +1620,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libgit2-sys" @@ -1757,9 +1757,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad" [[package]] name = "openssl-probe" @@ -2028,9 +2028,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -2126,9 +2126,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "ring" -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index e3062f96..d5825f14 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 4b7e1efb..3d7365c9 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.2 + productVersion: 3.5.5 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index c95898dc..09ae9893 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index f2f795d7..8c60fd38 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index cfd8addd..c1b3f124 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 04535100..9d54537c 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 37c09637..db399938 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index f392c62b..94153976 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -63,7 +63,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index 494d540b..e7d0e0cb 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -63,7 +63,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index 4100288d..1735246d 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable24.11.0 # <1> +FROM oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable25.3.0 # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" ---- @@ -75,8 +75,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1> - productVersion: "3.5.2" # <2> + custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev" # <1> + productVersion: "3.5.5" # <2> pullPolicy: IfNotPresent # <3> ... ---- @@ -134,7 +134,7 @@ spec: spark.sql.catalog.local.warehouse: /tmp/warehouse deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:1.3.1 # <1> + - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 # <1> ... ---- diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 3928356f..26a35f2e 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,5 +3,5 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) -- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) +- 3.5.5 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.5.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) diff --git a/examples/README-examples.md b/examples/README-examples.md index 307a6982..ced40e25 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.5.2.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.5.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.2.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.5.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/examples/ny-tlc-report-1.1.0-3.5.2.jar b/examples/ny-tlc-report-1.1.0-3.5.5.jar similarity index 90% rename from examples/ny-tlc-report-1.1.0-3.5.2.jar rename to examples/ny-tlc-report-1.1.0-3.5.5.jar index c88b521e..e5e16993 100644 Binary files a/examples/ny-tlc-report-1.1.0-3.5.2.jar and b/examples/ny-tlc-report-1.1.0-3.5.5.jar differ diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index 39cc12b8..3aaa2060 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 573f3da3..1578be11 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -7,7 +7,7 @@ metadata: spec: # everything under /jobs will be copied to /stackable/spark/jobs image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 - sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable0.0.0-dev + sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index 3481281c..45e918a6 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -13,9 +13,9 @@ metadata: name: spark-ny-cm spec: sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.2.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.5.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 4023fbc0..ed96f02d 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.2 + productVersion: 3.5.5 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs index d9a4e857..9d5c7ff5 100644 --- a/rust/operator-binary/src/crd/history.rs +++ b/rust/operator-binary/src/crd/history.rs @@ -453,7 +453,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.2 + productVersion: 3.5.5 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/history/config/jvm.rs b/rust/operator-binary/src/history/config/jvm.rs index eb7504ed..e241fcdb 100644 --- a/rust/operator-binary/src/history/config/jvm.rs +++ b/rust/operator-binary/src/history/config/jvm.rs @@ -65,7 +65,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.2 + productVersion: 3.5.5 logFileDirectory: s3: prefix: eventlogs/ @@ -98,7 +98,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.2 + productVersion: 3.5.5 logFileDirectory: s3: prefix: eventlogs/ diff --git a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 index 0311e042..84315812 100644 --- a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 @@ -48,7 +48,7 @@ spec: # # We extract the spark parts from the test scenario value. # - - org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark'].split('.')[:2]) }}_2.12:1.5.2 + - org.apache.iceberg:iceberg-spark-runtime-{{ ".".join(test_scenario['values']['spark'].split('.')[:2]) }}_2.12:1.8.1 volumes: - name: script configMap: diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.5.jar similarity index 90% rename from tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar rename to tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.5.jar index 7ea1fbf8..e5e16993 100644 Binary files a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.5.jar differ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index e9f98932..f25e8637 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,15 +5,15 @@ dimensions: - "false" - name: spark values: - - 3.5.1 - 3.5.2 + - 3.5.5 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev + # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev - name: spark-delta-lake values: - - 3.5.2 - # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev + - 3.5.5 + # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev - name: hbase values: - 2.6.0