diff --git a/CHANGELOG.md b/CHANGELOG.md index bb5f4280..0512060a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,13 @@ All notable changes to this project will be documented in this file. - Run a `containerdebug` process in the background of each Spark container to collect debugging information ([#508]). - Aggregate emitted Kubernetes events on the CustomResources ([#515]). +### Changed + +- Default to OCI for image metadata and product image selection ([#514]). + [#501]: https://github.com/stackabletech/spark-k8s-operator/pull/501 [#508]: https://github.com/stackabletech/spark-k8s-operator/pull/508 +[#514]: https://github.com/stackabletech/spark-k8s-operator/pull/514 [#515]: https://github.com/stackabletech/spark-k8s-operator/pull/515 ## [24.11.1] - 2025-01-10 diff --git a/Cargo.lock b/Cargo.lock index ea2128c5..b8b7b62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,8 +2331,8 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stackable-operator" -version = "0.84.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#b8fe43f7368249bf95b06d6cba3fd0135f7523ac" +version = "0.85.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" dependencies = [ "chrono", "clap", @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#b8fe43f7368249bf95b06d6cba3fd0135f7523ac" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" dependencies = [ "darling", "proc-macro2", @@ -2381,7 +2381,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#b8fe43f7368249bf95b06d6cba3fd0135f7523ac" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" dependencies = [ "kube", "semver", diff --git a/Cargo.nix b/Cargo.nix index efbee106..ba2f594b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -7229,13 +7229,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.84.1"; + version = "0.85.0"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b8fe43f7368249bf95b06d6cba3fd0135f7523ac"; - sha256 = "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i"; + rev = "59506c6202778889a27b6ae8153457e60a49c68d"; + sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; }; libName = "stackable_operator"; authors = [ @@ -7394,8 +7394,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b8fe43f7368249bf95b06d6cba3fd0135f7523ac"; - sha256 = "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i"; + rev = "59506c6202778889a27b6ae8153457e60a49c68d"; + sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -7429,8 +7429,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b8fe43f7368249bf95b06d6cba3fd0135f7523ac"; - sha256 = "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i"; + rev = "59506c6202778889a27b6ae8153457e60a49c68d"; + sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; }; libName = "stackable_shared"; authors = [ diff --git a/Cargo.toml b/Cargo.toml index c9e6c103..e377e329 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.84.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index a0346ddb..290d87f2 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,6 +1,6 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#stackable-operator-derive@0.3.1": "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#stackable-operator@0.84.1": "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.1#stackable-shared@0.0.1": "0vwq3dzxj56y4vrnw4ry7wajm12f32jipvc6f3izdrixy2pazq3i", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index e34ab310..096f6b47 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -917,7 +917,7 @@ spec: Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details. properties: custom: - description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` + description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -944,7 +944,7 @@ spec: nullable: true type: array repo: - description: Name of the docker repo, e.g. `docker.stackable.tech/stackable` + description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` nullable: true type: string stackableVersion: @@ -1047,7 +1047,7 @@ spec: Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details. properties: custom: - description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` + description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -1074,7 +1074,7 @@ spec: nullable: true type: array repo: - description: Name of the docker repo, e.g. `docker.stackable.tech/stackable` + description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` nullable: true type: string stackableVersion: diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 7ed08dd7..5d2a5845 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -1,7 +1,7 @@ # Default values for spark-k8s-operator. --- image: - repository: docker.stackable.tech/stackable/spark-k8s-operator + repository: oci.stackable.tech/sdp/spark-k8s-operator pullPolicy: IfNotPresent pullSecrets: [] diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 06dfe88a..f2f795d7 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-image namespace: default spec: - image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> + image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: productVersion: 3.5.2 mode: cluster diff --git a/docs/modules/spark-k8s/pages/reference/environment-variables.adoc b/docs/modules/spark-k8s/pages/reference/environment-variables.adoc index 2d5d97cd..d30c6f16 100644 --- a/docs/modules/spark-k8s/pages/reference/environment-variables.adoc +++ b/docs/modules/spark-k8s/pages/reference/environment-variables.adoc @@ -30,7 +30,7 @@ docker run \ --env KUBECONFIG=/home/stackable/.kube/config \ --env KUBERNETES_CLUSTER_DOMAIN=mycluster.local \ --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ -docker.stackable.tech/stackable/spark-k8s-operator:latest +oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev ---- == WATCH_NAMESPACE @@ -59,5 +59,5 @@ docker run \ --env KUBECONFIG=/home/stackable/.kube/config \ --env WATCH_NAMESPACE=test \ --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ -docker.stackable.tech/stackable/spark-k8s-operator:latest +oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev ---- diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index cbb1a604..4100288d 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable24.11.0 # <1> +FROM oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable24.11.0 # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" ---- @@ -75,7 +75,7 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1> + custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1> productVersion: "3.5.2" # <2> pullPolicy: IfNotPresent # <3> ... diff --git a/examples/README-examples.md b/examples/README-examples.md index 7d739f74..307a6982 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -16,7 +16,7 @@ stackablectl operator install spark-k8s commons secret Load the `ny-tlc-report` image to the cluster: ````text -kind load docker-image docker.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform +kind load docker-image oci.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform ```` ## Set up the `PersistentVolumeClaim` diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 64d13bae..573f3da3 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -6,8 +6,8 @@ metadata: namespace: default spec: # everything under /jobs will be copied to /stackable/spark/jobs - image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 - sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable0.0.0-dev + image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 + sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/kind/assert-pvc-jars.yaml b/kind/assert-pvc-jars.yaml index 3cc146f8..d8f43c2b 100644 --- a/kind/assert-pvc-jars.yaml +++ b/kind/assert-pvc-jars.yaml @@ -13,7 +13,7 @@ spec: claimName: pvc-ksv containers: - name: assert-pvc-jars - image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0 + image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0 env: - name: DEST_DIR value: "/dependencies/jars" diff --git a/kind/kind-pvc.yaml b/kind/kind-pvc.yaml index 69f558e2..e5e548ab 100644 --- a/kind/kind-pvc.yaml +++ b/kind/kind-pvc.yaml @@ -24,7 +24,7 @@ spec: claimName: pvc-ksv containers: - name: aws-deps - image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0 + image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0 env: - name: DEST_DIR value: "/dependencies/jars" diff --git a/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml b/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml index 2e3fa692..f3432ccf 100644 --- a/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml +++ b/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml @@ -10,7 +10,7 @@ spec: activeDeadlineSeconds: 100 containers: - name: history-api-check - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: [ "bash", diff --git a/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2 index 5d2eafcc..9804e8c1 100644 --- a/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2 @@ -12,7 +12,7 @@ spec: productVersion: "{{ test_scenario['values']['spark'] }}" {% endif %} pullPolicy: IfNotPresent - image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }} + image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }} vectorAggregatorConfigMapName: spark-vector-aggregator-discovery mode: cluster mainClass: org.apache.spark.examples.SparkALS diff --git a/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2 index a1338195..70d64f0c 100644 --- a/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2 @@ -29,7 +29,7 @@ spec: productVersion: "{{ test_scenario['values']['spark'] }}" {% endif %} pullPolicy: IfNotPresent - image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }} + image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }} vectorAggregatorConfigMapName: spark-vector-aggregator-discovery mode: cluster mainClass: org.apache.spark.examples.SparkALS diff --git a/tests/templates/kuttl/logging/10-install-test-runner.yaml b/tests/templates/kuttl/logging/10-install-test-runner.yaml index 5560d5d1..e10efb70 100644 --- a/tests/templates/kuttl/logging/10-install-test-runner.yaml +++ b/tests/templates/kuttl/logging/10-install-test-runner.yaml @@ -17,6 +17,6 @@ spec: spec: containers: - name: test-runner - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev stdin: true tty: true diff --git a/tests/templates/kuttl/overrides/20-test-logs.yaml b/tests/templates/kuttl/overrides/20-test-logs.yaml index 95c32ac9..60087002 100644 --- a/tests/templates/kuttl/overrides/20-test-logs.yaml +++ b/tests/templates/kuttl/overrides/20-test-logs.yaml @@ -10,7 +10,7 @@ spec: activeDeadlineSeconds: 100 containers: - name: history-api-check - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: [ "bash", diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 index 35aa5dc6..9d6f009e 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: pyspark-ny-public-s3-image spec: # everything under /jobs will be copied to /stackable/spark/jobs - image: "docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}" + image: "oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/60-test-logs.yaml b/tests/templates/kuttl/smoke/60-test-logs.yaml index 95c32ac9..60087002 100644 --- a/tests/templates/kuttl/smoke/60-test-logs.yaml +++ b/tests/templates/kuttl/smoke/60-test-logs.yaml @@ -10,7 +10,7 @@ spec: activeDeadlineSeconds: 100 containers: - name: history-api-check - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: [ "bash", diff --git a/tests/templates/kuttl/spark-history-server/20-test-logs.yaml b/tests/templates/kuttl/spark-history-server/20-test-logs.yaml index ea82e8ab..68c36402 100644 --- a/tests/templates/kuttl/spark-history-server/20-test-logs.yaml +++ b/tests/templates/kuttl/spark-history-server/20-test-logs.yaml @@ -10,7 +10,7 @@ spec: activeDeadlineSeconds: 100 containers: - name: history-api-check - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: [ "bash", diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index a25118e2..e9f98932 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -9,11 +9,11 @@ dimensions: - 3.5.2 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev + # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: spark-delta-lake values: - 3.5.2 - # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev + # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: hbase values: - 2.6.0