diff --git a/.scripts/update_refs.sh b/.scripts/update_refs.sh index abe1f622..6b2f57b2 100755 --- a/.scripts/update_refs.sh +++ b/.scripts/update_refs.sh @@ -35,10 +35,10 @@ function prepend { function maybe_commit { [ "$COMMIT" == "true" ] || return 0 local MESSAGE="$1" - PATCH=$(mktemp) + PATCH=$(mktemp --suffix=.diff) git add -u git diff --staged > "$PATCH" - git commit -S -m "$MESSAGE" --no-verify + git diff-index --quiet HEAD -- || git commit -S -m "$MESSAGE" --no-verify echo "patch written to: $PATCH" | prepend "\t" } @@ -55,7 +55,7 @@ if [[ "$CURRENT_BRANCH" == release-* ]]; then # Replace 0.0.0-dev refs with ${STACKABLE_RELEASE}.0 # TODO (@NickLarsenNZ): handle patches later, and what about release-candidates? - SEARCH='stackable(0\.0\.0-dev|24\.7\.[0-9]+)' # TODO (@NickLarsenNZ): After https://github.com/stackabletech/stackable-cockpit/issues/310, only search for 0.0.0-dev + SEARCH='stackable(0\.0\.0-dev)' REPLACEMENT="stackable${STACKABLE_RELEASE}.0" # TODO (@NickLarsenNZ): Be a bit smarter about patch releases. MESSAGE="Update image references with $REPLACEMENT" echo "$MESSAGE" diff --git a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml index dd650853..efa281b2 100644 --- a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml +++ b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: start-pyspark-job - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that diff --git a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml index b5e9ba8d..f74bab50 100644 --- a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml +++ b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: start-date-job - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml index 277c6005..6bcc924d 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml @@ -9,11 +9,11 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-kafka - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"] containers: - name: create-nifi-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 423f0fad..db28afc8 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -12,11 +12,11 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-kafka - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"] containers: - name: create-spark-ingestion-job - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"] volumeMounts: - name: manifest diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml index 8cb6e3da..6c0317bb 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml @@ -9,11 +9,11 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-testdata - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"] containers: - name: create-tables-in-trino - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml index d5fdff67..1f70f900 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-superset - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index 2c8c4df5..2e1bff2d 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-trino-tables - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: - bash - -euo @@ -23,7 +23,7 @@ spec: kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino containers: - name: create-spark-report - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: - bash - -euo diff --git a/demos/end-to-end-security/create-trino-tables.yaml b/demos/end-to-end-security/create-trino-tables.yaml index 7c488d5f..16b0e6f1 100644 --- a/demos/end-to-end-security/create-trino-tables.yaml +++ b/demos/end-to-end-security/create-trino-tables.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-tables-in-trino - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml b/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml index 7c561ed3..d250e0db 100644 --- a/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml +++ b/demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml @@ -9,7 +9,7 @@ spec: spec: containers: - name: create-hfile-and-import-to-hbase - image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.7.0 + image: docker.stackable.tech/stackable/hbase:2.4.18-stackable0.0.0-dev env: - name: HADOOP_USER_NAME value: stackable diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml index 3416ed91..93bedf98 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-druid-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"] volumeMounts: - name: ingestion-job-spec diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml index 231d8818..0d1aa1cc 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-nifi-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml index b52a2adf..36bc2110 100644 --- a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-superset - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml index 3c2d6208..88b8fea5 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-druid-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction"] volumeMounts: - name: ingestion-job-spec diff --git a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml index 6795a681..ff809981 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-nifi-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml index 6cf44c53..66f3ef56 100644 --- a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml +++ b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-superset - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index db643c3f..eb242ad3 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,3 +1,3 @@ -FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.7.0 +FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable0.0.0-dev RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar" diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml index 51179a50..f63712bc 100644 --- a/demos/signal-processing/create-nifi-ingestion-job.yaml +++ b/demos/signal-processing/create-nifi-ingestion-job.yaml @@ -9,13 +9,13 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-timescale-job - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for timescaleDB tables to be ready' && kubectl wait --for=condition=complete job/create-timescale-tables-job" ] containers: - name: create-nifi-ingestion-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "export PGPASSWORD=$(cat /timescale-admin-credentials/password) && \ curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml && \ sed -i \"s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g\" DownloadAndWriteToDB.xml && \ diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml index 61089f34..173639b0 100644 --- a/demos/signal-processing/create-timescale-tables.yaml +++ b/demos/signal-processing/create-timescale-tables.yaml @@ -9,7 +9,7 @@ spec: serviceAccountName: demo-serviceaccount initContainers: - name: wait-for-timescale - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for timescaleDB to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=postgresql-timescaledb" ] diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 5dce76c3..a4b46e46 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -8,11 +8,11 @@ spec: spec: initContainers: - name: wait-for-testdata - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"] containers: - name: create-spark-anomaly-detection-job - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"] volumeMounts: - name: manifest diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml index 36aba951..8a1a8cbc 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-superset - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/trino-taxi-data/create-table-in-trino.yaml b/demos/trino-taxi-data/create-table-in-trino.yaml index d45ce7d9..44e2500e 100644 --- a/demos/trino-taxi-data/create-table-in-trino.yaml +++ b/demos/trino-taxi-data/create-table-in-trino.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-ny-taxi-data-table-in-trino - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/trino-taxi-data/setup-superset.yaml b/demos/trino-taxi-data/setup-superset.yaml index 2c94efda..792b459b 100644 --- a/demos/trino-taxi-data/setup-superset.yaml +++ b/demos/trino-taxi-data/setup-superset.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-superset - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/docs/modules/demos/pages/airflow-scheduled-job.adoc b/docs/modules/demos/pages/airflow-scheduled-job.adoc index 1ecf3d80..933de6f1 100644 --- a/docs/modules/demos/pages/airflow-scheduled-job.adoc +++ b/docs/modules/demos/pages/airflow-scheduled-job.adoc @@ -1,5 +1,4 @@ = airflow-scheduled-job -:page-aliases: stable@stackablectl::demos/airflow-scheduled-job.adoc :description: This demo installs Airflow with Postgres and Redis on Kubernetes, showcasing DAG scheduling, job runs, and status verification via the Airflow UI. Install this demo on an existing Kubernetes cluster: diff --git a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc index f79dd363..96429e96 100644 --- a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc +++ b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc @@ -1,5 +1,4 @@ = data-lakehouse-iceberg-trino-spark -:page-aliases: stable@stackablectl::demos/data-lakehouse-iceberg-trino-spark.adoc :description: This demo shows a data workload with real-world data volumes using Trino, Kafka, Spark, NiFi, Superset and OPA. :demo-code: https://github.com/stackabletech/demos/blob/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml diff --git a/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc b/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc index 1d00ddd3..8421491c 100644 --- a/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc +++ b/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc @@ -1,5 +1,4 @@ = hbase-hdfs-cycling-data -:page-aliases: stable@stackablectl::demos/hbase-hdfs-load-cycling-data.adoc :description: Load cyclist data from HDFS to HBase on Kubernetes using Stackable's demo. Install, copy data, create HFiles, and query efficiently. :kaggle: https://www.kaggle.com/datasets/timgid/cyclistic-dataset-google-certificate-capstone?select=Divvy_Trips_2020_Q1.csv diff --git a/docs/modules/demos/pages/index.adoc b/docs/modules/demos/pages/index.adoc index 65ada08a..0e8b83cc 100644 --- a/docs/modules/demos/pages/index.adoc +++ b/docs/modules/demos/pages/index.adoc @@ -1,5 +1,4 @@ = Demos -:page-aliases: stable@stackablectl::demos/index.adoc :description: Explore Stackable demos showcasing data platform architectures. Includes external components for evaluation. The pages in this section guide you on how to use the demos provided by Stackable. diff --git a/docs/modules/demos/pages/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc b/docs/modules/demos/pages/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc index a732b82e..10f04d62 100644 --- a/docs/modules/demos/pages/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc +++ b/docs/modules/demos/pages/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc @@ -1,5 +1,4 @@ = jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data -:page-aliases: stable@stackablectl::demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc :scikit-lib: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html :k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu diff --git a/docs/modules/demos/pages/logging.adoc b/docs/modules/demos/pages/logging.adoc index 0466d0eb..8d28082b 100644 --- a/docs/modules/demos/pages/logging.adoc +++ b/docs/modules/demos/pages/logging.adoc @@ -1,5 +1,4 @@ = logging -:page-aliases: stable@stackablectl::demos/logging.adoc :description: Deploy a logging stack with OpenSearch, Vector, and Zookeeper for log data analysis using OpenSearch Dashboards in Kubernetes. :k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc index dff15bf5..f49a40c8 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc @@ -1,5 +1,4 @@ = nifi-kafka-druid-earthquake-data -:page-aliases: stable@stackablectl::demos/nifi-kafka-druid-earthquake-data.adoc :description: Install this demo for a showcase of using Kafka, Druid and Superset to view the global earthquake distribution. :superset-docs: https://superset.apache.org/docs/using-superset/creating-your-first-dashboard/#creating-charts-in-explore-view diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 0d25daa6..090ea66c 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -1,5 +1,4 @@ = nifi-kafka-druid-water-level-data -:page-aliases: stable@stackablectl::demos/nifi-kafka-druid-water-level-data.adoc :description: Install this demo for a showcase of using Kafka, Druid and Superset to visualize water levels in across Germany. :superset: https://superset.apache.org/docs/using-superset/creating-your-first-dashboard/#creating-charts-in-explore-view diff --git a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc index 319c523e..fcd12656 100644 --- a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc +++ b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc @@ -1,5 +1,4 @@ = spark-k8s-anomaly-detection-taxi-data -:page-aliases: stable@stackablectl::demos/spark-k8s-anomaly-detection-taxi-data.adoc :description: Deploy a Kubernetes-based Spark demo for anomaly detection using the popular New York taxi dataset, featuring Trino, Spark, MinIO, and Superset. :scikit-lib: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html diff --git a/docs/modules/demos/pages/trino-iceberg.adoc b/docs/modules/demos/pages/trino-iceberg.adoc index 34af036d..433e3a70 100644 --- a/docs/modules/demos/pages/trino-iceberg.adoc +++ b/docs/modules/demos/pages/trino-iceberg.adoc @@ -1,5 +1,4 @@ = trino-iceberg -:page-aliases: stable@stackablectl::demos/trino-iceberg.adoc :description: Install and explore Trino with Apache Iceberg for efficient SQL queries and scalable data management in a demo environment. :k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu diff --git a/docs/modules/demos/pages/trino-taxi-data.adoc b/docs/modules/demos/pages/trino-taxi-data.adoc index e8e9328f..541bb3ae 100644 --- a/docs/modules/demos/pages/trino-taxi-data.adoc +++ b/docs/modules/demos/pages/trino-taxi-data.adoc @@ -1,5 +1,4 @@ = trino-taxi-data -:page-aliases: stable@stackablectl::demos/trino-taxi-data.adoc :description: Install and demo Trino with NYC taxi data: Query with SQL, visualize with Superset, and explore data in MinIO and Trino on Kubernetes. :superset-docs: https://superset.apache.org/docs/creating-charts-dashboards/creating-your-first-dashboard#creating-charts-in-explore-view diff --git a/stacks/_templates/jupyterhub.yaml b/stacks/_templates/jupyterhub.yaml index fd4bbd81..05a28654 100644 --- a/stacks/_templates/jupyterhub.yaml +++ b/stacks/_templates/jupyterhub.yaml @@ -50,7 +50,7 @@ options: HADOOP_CONF_DIR: "/home/jovyan/hdfs" initContainers: - name: download-notebook - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb'] volumeMounts: - mountPath: /notebook diff --git a/stacks/_templates/keycloak.yaml b/stacks/_templates/keycloak.yaml index ecc9a9fb..345cf15a 100644 --- a/stacks/_templates/keycloak.yaml +++ b/stacks/_templates/keycloak.yaml @@ -48,7 +48,7 @@ spec: - name: tls mountPath: /tls/ - name: create-auth-class - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: ["/bin/bash", "-c"] args: - | diff --git a/stacks/end-to-end-security/krb5.yaml b/stacks/end-to-end-security/krb5.yaml index 19657231..3ce20bd4 100644 --- a/stacks/end-to-end-security/krb5.yaml +++ b/stacks/end-to-end-security/krb5.yaml @@ -14,7 +14,7 @@ spec: spec: initContainers: - name: init - image: docker.stackable.tech/stackable/krb5:1.21.1-stackable24.7.0 + image: docker.stackable.tech/stackable/krb5:1.21.1-stackable0.0.0-dev args: - sh - -euo @@ -35,7 +35,7 @@ spec: name: data containers: - name: kdc - image: docker.stackable.tech/stackable/krb5:1.21.1-stackable24.7.0 + image: docker.stackable.tech/stackable/krb5:1.21.1-stackable0.0.0-dev args: - krb5kdc - -n @@ -48,7 +48,7 @@ spec: - mountPath: /var/kerberos/krb5kdc name: data - name: kadmind - image: docker.stackable.tech/stackable/krb5:1.21.1-stackable24.7.0 + image: docker.stackable.tech/stackable/krb5:1.21.1-stackable0.0.0-dev args: - kadmind - -nofork @@ -61,7 +61,7 @@ spec: - mountPath: /var/kerberos/krb5kdc name: data - name: client - image: docker.stackable.tech/stackable/krb5:1.21.1-stackable24.7.0 + image: docker.stackable.tech/stackable/krb5:1.21.1-stackable0.0.0-dev tty: true stdin: true env: diff --git a/stacks/end-to-end-security/superset.yaml b/stacks/end-to-end-security/superset.yaml index 09ea404f..c606075c 100644 --- a/stacks/end-to-end-security/superset.yaml +++ b/stacks/end-to-end-security/superset.yaml @@ -25,7 +25,7 @@ spec: initContainers: # The postgres image does not contain curl or wget... - name: download-dump - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: - bash - -c diff --git a/stacks/keycloak-opa-poc/keycloak.yaml b/stacks/keycloak-opa-poc/keycloak.yaml index a6c2e225..eaf5d60b 100644 --- a/stacks/keycloak-opa-poc/keycloak.yaml +++ b/stacks/keycloak-opa-poc/keycloak.yaml @@ -70,7 +70,7 @@ spec: spec: containers: - name: propagate-keycloak-address - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev command: - bash - -x diff --git a/stacks/keycloak-opa-poc/setup-keycloak.yaml b/stacks/keycloak-opa-poc/setup-keycloak.yaml index f21d64a2..ceba42fe 100644 --- a/stacks/keycloak-opa-poc/setup-keycloak.yaml +++ b/stacks/keycloak-opa-poc/setup-keycloak.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: setup-keycloak - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev env: - name: KEYCLOAK_ADMIN_PASSWORD valueFrom: diff --git a/stacks/logging/setup-opensearch-dashboards.yaml b/stacks/logging/setup-opensearch-dashboards.yaml index c3b4330d..b7e00641 100644 --- a/stacks/logging/setup-opensearch-dashboards.yaml +++ b/stacks/logging/setup-opensearch-dashboards.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: setup-opensearch-dashboards - image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0 + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev env: - name: OPEN_SEARCH_ADMIN_PASSWORD valueFrom: diff --git a/stacks/signal-processing/jupyterhub.yaml b/stacks/signal-processing/jupyterhub.yaml index f26e5988..f9782df5 100644 --- a/stacks/signal-processing/jupyterhub.yaml +++ b/stacks/signal-processing/jupyterhub.yaml @@ -41,7 +41,7 @@ options: stackable.tech/vendor: Stackable initContainers: - name: download-notebook - image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0 + image: docker.stackable.tech/stackable/tools:1.0.0-stackable0.0.0-dev command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb'] volumeMounts: - mountPath: /notebook