chore(release): Backport relevant release-24.11 changes (#146)

* chore(release): Update stackableRelease to 24.11

* chore(release): Update image references with stackable24.11.0

* chore(release): Replace githubusercontent references main->release-24.11

* chore: Change docs version from nightly to 24.11

* chore: Remove superfluous page-aliases

* chore: Explicitly bump 24.11.0 to 24.11.1

These changes should be pulled into `main`, but this script needs more
work.

* chore(release): Update image references with stackable24.11.1

* fix(stack/end-to-end-security): Skip DB restore if the DB exists (#139)

Otherwise it breaks with:

```
ERROR [flask_migrate] Error: Requested revision 17fcea065655 overlaps with other requested revisions b7851ee5522f
```

The latter revision being the one that exists in the uploaded dump.

* change references back to main etc.

* prepare update_refs for next release

* typo

* revert changes for products

* set images back to dev except 24.3.0 exceptions

* Update .scripts/update_refs.sh

Co-authored-by: Nick <[email protected]>

---------

Co-authored-by: Techassi <[email protected]>
Co-authored-by: Techassi <[email protected]>
Co-authored-by: Nick Larsen <[email protected]>
Co-authored-by: Nick <[email protected]>

Author: 3 people

.scripts/update_refs.sh

@@ -35,10 +35,10 @@ function prepend {
 function maybe_commit {
   [ "$COMMIT" == "true" ] || return 0
   local MESSAGE="$1"
-  PATCH=$(mktemp)
+  PATCH=$(mktemp --suffix=.diff)
   git add -u
   git diff --staged > "$PATCH"
-  git commit -S -m "$MESSAGE" --no-verify
+  git diff-index --quiet HEAD -- || git commit -S -m "$MESSAGE" --no-verify
   echo "patch written to: $PATCH" | prepend "\t"
 }
 
@@ -55,7 +55,7 @@ if [[ "$CURRENT_BRANCH" == release-* ]]; then
 
   # Replace 0.0.0-dev refs with ${STACKABLE_RELEASE}.0
   # TODO (@NickLarsenNZ): handle patches later, and what about release-candidates?
-  SEARCH='stackable(0\.0\.0-dev|24\.7\.[0-9]+)' # TODO (@NickLarsenNZ): After https://github.com/stackabletech/stackable-cockpit/issues/310, only search for 0.0.0-dev
+  SEARCH='stackable(0\.0\.0-dev)'
   REPLACEMENT="stackable${STACKABLE_RELEASE}.0" # TODO (@NickLarsenNZ): Be a bit smarter about patch releases.
   MESSAGE="Update image references with $REPLACEMENT"
   echo "$MESSAGE"

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-pyspark-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-date-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -12,11 +12,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
       containers:
         - name: create-spark-ingestion-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
           volumeMounts:
             - name: manifest

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev
           command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/end-to-end-security/create-spark-report.yaml

@@ -12,7 +12,7 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-trino-tables
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             - bash
             - -euo
@@ -23,7 +23,7 @@ spec:
               kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino
       containers:
         - name: create-spark-report
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             - bash
             - -euo

demos/end-to-end-security/create-trino-tables.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: create-hfile-and-import-to-hbase
-          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.7.0
+          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable0.0.0-dev
           env:
             - name: HADOOP_USER_NAME
               value: stackable