chore: Backport relevant release 24.11 changes

.scripts/update_refs.sh

@@ -35,10 +35,10 @@ function prepend {
 function maybe_commit {
   [ "$COMMIT" == "true" ] || return 0
   local MESSAGE="$1"
-  PATCH=$(mktemp)
+  PATCH=$(mktemp --suffix=.diff)
   git add -u
   git diff --staged > "$PATCH"
-  git commit -S -m "$MESSAGE" --no-verify
+  git diff-index --quiet HEAD -- || git commit -S -m "$MESSAGE" --no-verify
   echo "patch written to: $PATCH" | prepend "\t"
 }
 
@@ -55,8 +55,8 @@ if [[ "$CURRENT_BRANCH" == release-* ]]; then
 
   # Replace 0.0.0-dev refs with ${STACKABLE_RELEASE}.0
   # TODO (@NickLarsenNZ): handle patches later, and what about release-candidates?
-  SEARCH='stackable(0\.0\.0-dev|24\.7\.[0-9]+)' # TODO (@NickLarsenNZ): After https://github.com/stackabletech/stackable-cockpit/issues/310, only search for 0.0.0-dev
-  REPLACEMENT="stackable${STACKABLE_RELEASE}.0" # TODO (@NickLarsenNZ): Be a bit smarter about patch releases.
+  SEARCH='stackable(0\.0\.0-dev|24\.7\.[0-9]+|24\.11\.0)' # TODO (@NickLarsenNZ): After https://github.com/stackabletech/stackable-cockpit/issues/310, only search for 0.0.0-dev
+  REPLACEMENT="stackable${STACKABLE_RELEASE}.1" # TODO (@NickLarsenNZ): Be a bit smarter about patch releases.
   MESSAGE="Update image references with $REPLACEMENT"
   echo "$MESSAGE"
   find demos stacks -type f \

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-pyspark-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-date-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -9,12 +9,12 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
-          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
+          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -12,11 +12,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
       containers:
         - name: create-spark-ingestion-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
           volumeMounts:
             - name: manifest

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.11.1
           command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -8,8 +8,8 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
+          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/demos-v2.yaml

@@ -7,10 +7,10 @@ demos:
       - airflow
       - job-scheduling
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 2401m
@@ -24,8 +24,8 @@ demos:
       - hdfs
       - cycling-tripdata
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: "3"
@@ -43,9 +43,9 @@ demos:
       - opa
       - keycloak
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/create-trino-tables.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/serviceaccount.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/end-to-end-security/create-spark-report.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/end-to-end-security/create-trino-tables.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/end-to-end-security/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/end-to-end-security/create-spark-report.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 9000m
@@ -64,9 +64,9 @@ demos:
       - s3
       - earthquakes
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
     supportedNamespaces: ["default"]
     resourceRequests:
       cpu: 8700m
@@ -85,9 +85,9 @@ demos:
       - s3
       - water-levels
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
     supportedNamespaces: ["default"]
     resourceRequests:
       cpu: 8900m
@@ -104,10 +104,10 @@ demos:
       - s3
       - ny-taxi-data
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 6400m
@@ -139,9 +139,9 @@ demos:
       - s3
       - ny-taxi-data
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/load-test-data.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/create-table-in-trino.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/setup-superset.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/trino-taxi-data/load-test-data.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/trino-taxi-data/create-table-in-trino.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/trino-taxi-data/setup-superset.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 6800m
@@ -164,12 +164,12 @@ demos:
       - water-levels
       - earthquakes
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
     supportedNamespaces: ["default"]
     resourceRequests:
       cpu: "80"
@@ -185,7 +185,7 @@ demos:
       - pyspark
       - ny-taxi-data
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 3350m
@@ -202,7 +202,7 @@ demos:
       - vector
       - zookeeper
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/logging/zookeeper.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/logging/zookeeper.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: 6500m
@@ -218,9 +218,9 @@ demos:
       - grafana-dashboards
       - zookeeper
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/serviceaccount.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/create-timescale-tables.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/create-nifi-ingestion-job.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/signal-processing/serviceaccount.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/signal-processing/create-timescale-tables.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/release-24.11/demos/signal-processing/create-nifi-ingestion-job.yaml
     supportedNamespaces: []
     resourceRequests:
       cpu: "3"

demos/end-to-end-security/create-spark-report.yaml

@@ -12,7 +12,7 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-trino-tables
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
           command:
             - bash
             - -euo
@@ -23,7 +23,7 @@ spec:
               kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino
       containers:
         - name: create-spark-report
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
           command:
             - bash
             - -euo

demos/end-to-end-security/create-trino-tables.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: create-hfile-and-import-to-hbase
-          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.7.0
+          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.11.1
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: load-ny-taxi-data
-          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.11.1
           # yamllint disable rule:line-length
           command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
           && cd /tmp \

demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-druid-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.11.1
           command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"]
           volumeMounts:
             - name: ingestion-job-spec