refactor: Release 24.3 changes (#24)

* feat(stack/hdfs-hbase): upgade zookeeper

* refactor(stack/hdfs-hbase): move listenerClass configs to appropriate roles

* feat(stack/hdfs-hbase): add listener operator

* docs(demo/hbase-hdfs-load-cycling-data): improve readability

* docs(demo/hbase-hdfs-load-cycling-data): improve instructions

* chore(stack/template): update jupyterlab template

* chore(stack/jupyterhub-pyspark-hdfs): update pyspark image

* chore(demo/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data): update image tags

* chore(demo/hbase-hdfs-load-cycling-data): update image tags

* chore(stack/data-lakehouse-iceberg-trino-spark): update productVersion

* docs(demo/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data): improve descriptions

* docs(demo/hbase-hdfs-load-cycling-data): improve readability

* docs(demo/data-lakehouse-iceberg-trino-spark): add install instructions

* docs(demo/dairflow-scheduled-job): add install instructions

* docs: add install command, move system requirements up, sentence case titles

* docs(demo/logging): reorganise docs, update images

* docs: add install command, move system requirements up, sentence case titles

* chore(demo/logging): bump zookeeper to 3.9.1

* chore(demo/logging): bump testing tools to 0.0.0-dev

* chore(stack/jupyterhub-pyspark-hdfs): move hdfs listenerClass, point to new image in notebook, bump zookeeper to 3.9.1

* chore(stack/dual-hive-hdfs-s3): bump trino to 440

* chore(stack/data-lakehouse-iceberg-trino-spark): bump trino to 440

* chore(stack/templates): bump opensearch

* chore(demo/nifi-water: bumped products etc.

* chore(demo/spark-anomaly-detection): updates to products and dependencies

* chore(demo/spark-anomaly-detection): updates to images

* all the docs

* chore(stack/template): move jupyter template back to python3.9, use notebook on branch

* chore(demo/data-lakehouse-iceberg-trino-spark): use 0.0.0-dev images, bump minio image

* chore(demo/nifi-water): remove leading space from urls

* chore(demo/nifi-earthquake): bump test images

* chore(demo/trino-taxi-data): update docs, screenshots, rego rules and stack (trino 442)

* chore(demo/trino-taxi-data): restore docs section that still makes sense

* chore(demo/signal-processing): bump Nifi, update Grafana and Jupyterlab

* bump zookeepers to 3.9.2

* bump airflow to 2.8.1

* chore: update all stacks and images to release 24.3

* fix: image version in jupyter notebook

---------

Co-authored-by: Andrew Kenworthy <[email protected]>
Co-authored-by: Razvan-Daniel Mihai <[email protected]>

Author: 3 people

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-pyspark-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-date-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -12,11 +12,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-spark-ingestion-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
           volumeMounts:
             - name: manifest

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
           command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: load-test-data
-          image: "bitnami/minio:2022-debian-10"
+          image: "bitnami/minio:2024-debian-12"
           # Please try to order the load jobs from small to large datasets
           command: ["bash", "-c", "mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && cd /tmp && curl -O https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv && mc cp open-postcode-geo.csv minio/staging/house-sales/postcode-geo-lookup/ && rm open-postcode-geo.csv && for year in {2005..2021}; do curl -O https://repo.stackable.tech/repository/misc/datasets/uk-house-sales/uk-house-sales-$year.csv && mc cp uk-house-sales-$year.csv minio/staging/house-sales/house-sales/ && rm uk-house-sales-$year.csv; done && curl -O https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv && mc cp earthquakes_1950_to_2022.csv minio/staging/earthquakes/earthquakes/ && rm earthquakes_1950_to_2022.csv && curl -O https://repo.stackable.tech/repository/misc/datasets/e-charging-stations/e-charging-stations-2022-08.csv && mc cp e-charging-stations-2022-08.csv minio/staging/smart-city/e-charging-stations/ && rm e-charging-stations-2022-08.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/taxi_zone_lookup.csv && mc cp taxi_zone_lookup.csv minio/staging/taxi/taxi-zone-lookup/ && rm taxi_zone_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/rate_code_lookup.csv && mc cp rate_code_lookup.csv minio/staging/taxi/rate-code-lookup/ && rm rate_code_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/payment_type_lookup.csv && mc cp payment_type_lookup.csv minio/staging/taxi/payment-type-lookup/ && rm payment_type_lookup.csv && for month in 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/green_tripdata_$month.parquet && mc cp green_tripdata_$month.parquet minio/staging/taxi/green-tripdata/ && rm green_tripdata_$month.parquet; done && for month in 2015-01 2015-02 2015-03 2015-04 2015-05 2015-06 2015-07 2015-08 2015-09 2015-10 2015-11 2015-12 2016-01 2016-02 2016-03 2016-04 2016-05 2016-06 2016-07 2016-08 2016-09 2016-10 2016-11 2016-12 2017-01 2017-02 2017-03 2017-04 2017-05 2017-06 2017-07 2017-08 2017-09 2017-10 2017-11 2017-12 2018-01 2018-02 2018-03 2018-04 2018-05 2018-06 2018-07 2018-08 2018-09 2018-10 2018-11 2018-12 2019-01 2019-02 2019-03 2019-04 2019-05 2019-06 2019-07 2019-08 2019-09 2019-10 2019-11 2019-12 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06 2022-07 2022-08 2022-09 2022-10 2022-11 2022-12 2023-01 2023-02 2023-03 2023-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc cp yellow_tripdata_$month.parquet minio/staging/taxi/yellow-tripdata/ && rm yellow_tripdata_$month.parquet; done && for month in 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc cp fhvhv_tripdata_$month.parquet minio/staging/taxi/fhvhv-tripdata/ && rm fhvhv_tripdata_$month.parquet; done"]
           volumeMounts:

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: create-hfile-and-import-to-hbase
-          image: docker.stackable.tech/stackable/hbase:2.4.17-stackable23.11.0
+          image: docker.stackable.tech/stackable/hbase:2.4.17-stackable24.3.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: distcp-cycling-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable23.11.0
+          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable24.3.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile

@@ -1,6 +1,6 @@
-# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable23.4 .
+# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable0.0.0-dev .
 
-FROM docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable23.4
+FROM docker.stackable.tech/stackable/pyspark-k8s:3.5.0-stackable24.3.0
 
 COPY requirements.txt .