From 42c38c1f74163740378235e18e33d45c6e127531 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 5 Aug 2024 18:36:08 +0200 Subject: [PATCH 01/20] chore(opa): bump to 0.67.0 Depends on: https://github.com/stackabletech/docker-images/pull/797 --- stacks/data-lakehouse-iceberg-trino-spark/trino.yaml | 2 +- stacks/end-to-end-security/opa.yaml | 2 +- stacks/keycloak-opa-poc/opa.yaml | 2 +- stacks/trino-iceberg/trino.yaml | 2 +- stacks/trino-superset-s3/trino.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 10274bd2..fa86633f 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -114,7 +114,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml index 543a8f06..23ca47fe 100644 --- a/stacks/end-to-end-security/opa.yaml +++ b/stacks/end-to-end-security/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 clusterConfig: userInfo: backend: diff --git a/stacks/keycloak-opa-poc/opa.yaml b/stacks/keycloak-opa-poc/opa.yaml index 41615844..896c158b 100644 --- a/stacks/keycloak-opa-poc/opa.yaml +++ b/stacks/keycloak-opa-poc/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index 4f15796a..029167e2 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -99,7 +99,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index c8df87b2..2fe772c9 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -71,7 +71,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} From 28afbe848dea27fa7abc82148c7e32fae91cd629 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 6 Aug 2024 12:30:14 +0200 Subject: [PATCH 02/20] chore(opa): bump 0.67.1 --- stacks/data-lakehouse-iceberg-trino-spark/trino.yaml | 2 +- stacks/end-to-end-security/opa.yaml | 2 +- stacks/keycloak-opa-poc/opa.yaml | 2 +- stacks/trino-iceberg/trino.yaml | 2 +- stacks/trino-superset-s3/trino.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index fa86633f..687cad42 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -114,7 +114,7 @@ metadata: name: opa spec: image: - productVersion: 0.67.0 + productVersion: 0.67.1 servers: roleGroups: default: {} diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml index 23ca47fe..ae655c60 100644 --- a/stacks/end-to-end-security/opa.yaml +++ b/stacks/end-to-end-security/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.67.0 + productVersion: 0.67.1 clusterConfig: userInfo: backend: diff --git a/stacks/keycloak-opa-poc/opa.yaml b/stacks/keycloak-opa-poc/opa.yaml index 896c158b..bcd2919f 100644 --- a/stacks/keycloak-opa-poc/opa.yaml +++ b/stacks/keycloak-opa-poc/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.67.0 + productVersion: 0.67.1 servers: roleGroups: default: {} diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index 029167e2..e725ef22 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -99,7 +99,7 @@ metadata: name: opa spec: image: - productVersion: 0.67.0 + productVersion: 0.67.1 servers: roleGroups: default: diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index 2fe772c9..13596e87 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -71,7 +71,7 @@ metadata: name: opa spec: image: - productVersion: 0.67.0 + productVersion: 0.67.1 servers: roleGroups: default: {} From d3bec37ca0ab2944c1b2fbaef1da045b9c1cefca Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 5 Aug 2024 18:36:08 +0200 Subject: [PATCH 03/20] chore(opa): bump to 0.67.0 Depends on: https://github.com/stackabletech/docker-images/pull/797 --- stacks/data-lakehouse-iceberg-trino-spark/trino.yaml | 2 +- stacks/end-to-end-security/opa.yaml | 2 +- stacks/keycloak-opa-poc/opa.yaml | 2 +- stacks/trino-iceberg/trino.yaml | 2 +- stacks/trino-superset-s3/trino.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 10274bd2..fa86633f 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -114,7 +114,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml index 96e4dec9..212ad28a 100644 --- a/stacks/end-to-end-security/opa.yaml +++ b/stacks/end-to-end-security/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 clusterConfig: userInfo: backend: diff --git a/stacks/keycloak-opa-poc/opa.yaml b/stacks/keycloak-opa-poc/opa.yaml index 41615844..896c158b 100644 --- a/stacks/keycloak-opa-poc/opa.yaml +++ b/stacks/keycloak-opa-poc/opa.yaml @@ -5,7 +5,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index 4f15796a..029167e2 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -99,7 +99,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index c8df87b2..2fe772c9 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -71,7 +71,7 @@ metadata: name: opa spec: image: - productVersion: 0.66.0 + productVersion: 0.67.0 servers: roleGroups: default: {} From 7bf3c33190f245b0c991ebba71e2569241e0f352 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 8 Aug 2024 14:15:20 +0200 Subject: [PATCH 04/20] chore: Bump Vector to 0.40.0 --- stacks/_templates/vector-aggregator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml index d7dd458a..cdedc2ee 100644 --- a/stacks/_templates/vector-aggregator.yaml +++ b/stacks/_templates/vector-aggregator.yaml @@ -3,7 +3,7 @@ name: vector repo: name: vector url: https://helm.vector.dev -version: 0.34.0 +version: 0.35.0 # app version 0.40.0 options: commonLabels: stackable.tech/vendor: Stackable From 890df16374b9726609632d7b32afab35fac71ba4 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 26 Aug 2024 21:05:34 +0200 Subject: [PATCH 05/20] chore(airflow): bump version to 2.9.3 --- stacks/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 03d2b278..e1f0a68a 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -6,7 +6,7 @@ metadata: name: airflow spec: image: - productVersion: 2.9.2 + productVersion: 2.9.3 clusterConfig: listenerClass: external-unstable loadExamples: false From e5da07dd424efbbb7e0c82e9afd0f913bc98f637 Mon Sep 17 00:00:00 2001 From: xeniape Date: Thu, 5 Sep 2024 16:52:47 +0200 Subject: [PATCH 06/20] chore(trino): bump version to 455 --- docs/modules/demos/pages/trino-iceberg.adoc | 2 +- stacks/data-lakehouse-iceberg-trino-spark/trino.yaml | 2 +- stacks/dual-hive-hdfs-s3/trino.yaml | 2 +- stacks/end-to-end-security/trino-regorules.yaml | 8 ++++---- stacks/end-to-end-security/trino.yaml | 2 +- stacks/keycloak-opa-poc/trino.yaml | 2 +- stacks/trino-iceberg/trino.yaml | 2 +- stacks/trino-superset-s3/trino.yaml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/modules/demos/pages/trino-iceberg.adoc b/docs/modules/demos/pages/trino-iceberg.adoc index ef9601b6..fcc283e4 100644 --- a/docs/modules/demos/pages/trino-iceberg.adoc +++ b/docs/modules/demos/pages/trino-iceberg.adoc @@ -82,7 +82,7 @@ connect with DBeaver. As an alternative, you can use https://trino.io/download.h [source,console] ---- -$ java -jar ~/Downloads/trino-cli-451-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856 +$ java -jar ~/Downloads/trino-cli-455-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856 ---- Make sure to replace the server endpoint with the endpoint listed in the `stackablectl stacklet list` output. When diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 687cad42..6ba799fd 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -5,7 +5,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: listenerClass: external-unstable catalogLabelSelector: diff --git a/stacks/dual-hive-hdfs-s3/trino.yaml b/stacks/dual-hive-hdfs-s3/trino.yaml index 0ac7bb9f..9aff8a9c 100644 --- a/stacks/dual-hive-hdfs-s3/trino.yaml +++ b/stacks/dual-hive-hdfs-s3/trino.yaml @@ -61,7 +61,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: authorization: opa: diff --git a/stacks/end-to-end-security/trino-regorules.yaml b/stacks/end-to-end-security/trino-regorules.yaml index cb45f4e8..fdd55d87 100644 --- a/stacks/end-to-end-security/trino-regorules.yaml +++ b/stacks/end-to-end-security/trino-regorules.yaml @@ -1460,7 +1460,7 @@ data: # "groups": ["group1", ...], # "user": "username", # }, - # "softwareStack": {"trinoVersion": "442"}, + # "softwareStack": {"trinoVersion": "455"}, # } # } # @@ -1610,7 +1610,7 @@ data: # "groups": ["group1", ...], # "user": "username", # }, - # "softwareStack": {"trinoVersion": "442"}, + # "softwareStack": {"trinoVersion": "455"}, # } # } # @@ -1671,7 +1671,7 @@ data: # "groups": ["group1", ...], # "user": "username", # }, - # "softwareStack": {"trinoVersion": "442"}, + # "softwareStack": {"trinoVersion": "455"}, # } # } # @@ -1736,7 +1736,7 @@ data: # "groups": ["group1", ...], # "user": "username", # }, - # "softwareStack": {"trinoVersion": "442"}, + # "softwareStack": {"trinoVersion": "455"}, # } # } # diff --git a/stacks/end-to-end-security/trino.yaml b/stacks/end-to-end-security/trino.yaml index 4aebad31..5f5d4f0a 100644 --- a/stacks/end-to-end-security/trino.yaml +++ b/stacks/end-to-end-security/trino.yaml @@ -5,7 +5,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: listenerClass: external-unstable tls: diff --git a/stacks/keycloak-opa-poc/trino.yaml b/stacks/keycloak-opa-poc/trino.yaml index 315ec5b0..97a6453a 100644 --- a/stacks/keycloak-opa-poc/trino.yaml +++ b/stacks/keycloak-opa-poc/trino.yaml @@ -5,7 +5,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: listenerClass: external-unstable tls: diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index e725ef22..b826b798 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -5,7 +5,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: listenerClass: external-unstable catalogLabelSelector: diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index 13596e87..04a61a7e 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -5,7 +5,7 @@ metadata: name: trino spec: image: - productVersion: "451" + productVersion: "455" clusterConfig: listenerClass: external-unstable catalogLabelSelector: From a53b25bcbdb8a7a2549416481233b9871974bb1f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 11 Sep 2024 18:07:20 +0200 Subject: [PATCH 07/20] chore(spark): version 3.5.2 --- .../create-spark-ingestion-job.yaml | 4 ++-- demos/end-to-end-security/create-spark-report.yaml | 2 +- .../create-spark-anomaly-detection-job.yaml | 2 +- stacks/airflow/airflow.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 8915daf5..117702c5 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -43,13 +43,13 @@ data: stackable.tech/vendor: Stackable spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py deps: packages: - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.0 - - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1 + - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2 s3connection: reference: minio sparkConf: diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index c256f2b1..c72845e5 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -55,7 +55,7 @@ data: name: spark-report spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/spark-report.py deps: diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 39cf03a3..5dce76c3 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -37,7 +37,7 @@ data: name: spark-ad spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///spark-scripts/spark-ad.py deps: diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index e1f0a68a..5e8b96ce 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -272,7 +272,7 @@ data: spec: version: "1.0" sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py job: From 972eafd2d0efee5f398a38fbbcdda1d50a9a78af Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 12 Sep 2024 06:21:22 -0400 Subject: [PATCH 08/20] Update create-spark-ingestion-job.yaml --- .../create-spark-ingestion-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 117702c5..813aea2c 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -48,7 +48,7 @@ data: mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.0 + - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1 - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2 s3connection: reference: minio From af5654b951b4c824856d219028a40bb971eff6ef Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:26:20 +0200 Subject: [PATCH 09/20] chore: bump hdfs version to 3.4.0 --- .../hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml | 3 ++- .../load-test-data.yaml | 4 +++- .../load-test-data.yaml | 1 + demos/trino-taxi-data/load-test-data.yaml | 1 + stacks/dual-hive-hdfs-s3/hdfs.yaml | 2 +- stacks/end-to-end-security/hdfs.yaml | 8 +++++--- stacks/hdfs-hbase/hdfs.yaml | 3 ++- stacks/jupyterhub-pyspark-hdfs/hdfs.yaml | 2 +- stacks/keycloak-opa-poc/hdfs.yaml | 2 +- 9 files changed, 17 insertions(+), 9 deletions(-) diff --git a/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml b/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml index d487b8a6..44300127 100644 --- a/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml +++ b/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml @@ -11,7 +11,7 @@ spec: # We use 24.3.0 here which contains the distcp MapReduce components # This is not included in the 24.7 images and will fail. # See: https://github.com/stackabletech/docker-images/issues/793 - image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.3.0 + image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.11.0 env: - name: HADOOP_USER_NAME value: stackable @@ -19,6 +19,7 @@ spec: value: "/stackable/conf/hdfs" - name: HADOOP_CLASSPATH value: "/stackable/hadoop/share/hadoop/tools/lib/*.jar" + # yamllint disable-line rule:line-length command: ["bash", "-c", "bin/hdfs dfs -mkdir -p /data/raw && bin/hadoop distcp -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider s3a://public-backup-nyc-tlc/cycling-tripdata/demo-cycling-tripdata.csv.gz hdfs://hdfs/data/raw"] volumeMounts: - name: config-volume-hdfs diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml index 7add3113..7dd44f89 100644 --- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml @@ -8,13 +8,15 @@ spec: spec: containers: - name: load-ny-taxi-data - image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.7.0 + image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.11.0 + # yamllint disable rule:line-length command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \ && cd /tmp \ && for month in 2020-09; do \ curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet \ && /stackable/hadoop/bin/hdfs dfs -put fhvhv_tripdata_$month.parquet /ny-taxi-data/raw/; \ done"] + # yamllint enable rule:line-length volumeMounts: - name: hdfs-discovery-configmap mountPath: /hdfs diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index 395249eb..299d473a 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -9,6 +9,7 @@ spec: containers: - name: load-ny-taxi-data image: "bitnami/minio:2022-debian-10" + # yamllint disable-line rule:line-length command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"] volumeMounts: - name: minio-s3-credentials diff --git a/demos/trino-taxi-data/load-test-data.yaml b/demos/trino-taxi-data/load-test-data.yaml index d0639ab6..4afe0d58 100644 --- a/demos/trino-taxi-data/load-test-data.yaml +++ b/demos/trino-taxi-data/load-test-data.yaml @@ -9,6 +9,7 @@ spec: containers: - name: load-ny-taxi-data image: "bitnami/minio:2024-debian-12" + # yamllint disable-line rule:line-length command: ["bash", "-c", "cd /tmp && for month in 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp yellow_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/; done"] volumeMounts: - name: minio-s3-credentials diff --git a/stacks/dual-hive-hdfs-s3/hdfs.yaml b/stacks/dual-hive-hdfs-s3/hdfs.yaml index 386e8793..5c6b6d83 100644 --- a/stacks/dual-hive-hdfs-s3/hdfs.yaml +++ b/stacks/dual-hive-hdfs-s3/hdfs.yaml @@ -25,7 +25,7 @@ metadata: name: hdfs spec: image: - productVersion: 3.3.4 + productVersion: 3.4.0 clusterConfig: listenerClass: external-unstable dfsReplication: 1 diff --git a/stacks/end-to-end-security/hdfs.yaml b/stacks/end-to-end-security/hdfs.yaml index 0330e001..e976e21e 100644 --- a/stacks/end-to-end-security/hdfs.yaml +++ b/stacks/end-to-end-security/hdfs.yaml @@ -1,10 +1,11 @@ +--- apiVersion: hdfs.stackable.tech/v1alpha1 kind: HdfsCluster metadata: name: hdfs spec: image: - productVersion: 3.3.4 + productVersion: 3.4.0 clusterConfig: zookeeperConfigMapName: hdfs-znode authentication: @@ -29,8 +30,9 @@ spec: level: DEBUG configOverrides: &configOverrides core-site.xml: - # The idea is that the user "hive" can't do anything in hdfs, *but* it can impersonate other users - # (such as trino), that have the needed permissions + # The idea is that the user "hive" can't do anything in hdfs, + # *but* it can impersonate other users (such as trino), + # that have the needed permissions hadoop.proxyuser.hive.users: "*" hadoop.proxyuser.hive.hosts: "*" roleGroups: diff --git a/stacks/hdfs-hbase/hdfs.yaml b/stacks/hdfs-hbase/hdfs.yaml index 07649b3b..c65a2b9a 100644 --- a/stacks/hdfs-hbase/hdfs.yaml +++ b/stacks/hdfs-hbase/hdfs.yaml @@ -1,10 +1,11 @@ +--- apiVersion: hdfs.stackable.tech/v1alpha1 kind: HdfsCluster metadata: name: hdfs spec: image: - productVersion: 3.3.4 + productVersion: 3.4.0 clusterConfig: dfsReplication: 1 zookeeperConfigMapName: hdfs-znode diff --git a/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml b/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml index f7835b70..54b0ad69 100644 --- a/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml +++ b/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml @@ -13,7 +13,7 @@ metadata: name: hdfs spec: image: - productVersion: 3.3.4 + productVersion: 3.4.0 clusterConfig: dfsReplication: 1 zookeeperConfigMapName: hdfs-znode diff --git a/stacks/keycloak-opa-poc/hdfs.yaml b/stacks/keycloak-opa-poc/hdfs.yaml index 05eb35d5..30222c36 100644 --- a/stacks/keycloak-opa-poc/hdfs.yaml +++ b/stacks/keycloak-opa-poc/hdfs.yaml @@ -5,7 +5,7 @@ metadata: name: hdfs spec: image: - productVersion: 3.3.4 + productVersion: 3.4.0 clusterConfig: dfsReplication: 1 zookeeperConfigMapName: hdfs-znode From d5b4b252cc245190e00b4490f4bf7d149483fdb4 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:57:47 +0200 Subject: [PATCH 10/20] chore: bump nifi for 24.11 --- demos/signal-processing/Dockerfile-nifi | 2 +- demos/signal-processing/DownloadAndWriteToDB.xml | 12 ++++++------ stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index be1c2594..052cacd2 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,5 +1,5 @@ # docker build -f ./Dockerfile-nifi -t docker.stackable.tech/demos/nifi:1.27.0-postgresql . -FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.7.0 +FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.11.0 RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar" diff --git a/demos/signal-processing/DownloadAndWriteToDB.xml b/demos/signal-processing/DownloadAndWriteToDB.xml index 6bacabef..28fcb414 100644 --- a/demos/signal-processing/DownloadAndWriteToDB.xml +++ b/demos/signal-processing/DownloadAndWriteToDB.xml @@ -61,7 +61,7 @@ nifi-dbcp-service-nar org.apache.nifi - 1.21.0 + 1.27.0 @@ -258,7 +258,7 @@ nifi-record-serialization-services-nar org.apache.nifi - 1.21.0 + 1.27.0 @@ -561,7 +561,7 @@ 88.0 @@ -584,7 +584,7 @@ through "future" data that has already been persisted. nifi-standard-nar org.apache.nifi - 1.21.0 + 1.27.0 PENALIZE_FLOWFILE @@ -1069,7 +1069,7 @@ through "future" data that has already been persisted. nifi-standard-nar org.apache.nifi - 1.21.0 + 1.27.0 PENALIZE_FLOWFILE @@ -1223,7 +1223,7 @@ from conditions_temp; nifi-standard-nar org.apache.nifi - 1.21.0 + 1.27.0 PENALIZE_FLOWFILE diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml index 20a8c805..c59001b7 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml @@ -5,7 +5,7 @@ metadata: name: nifi spec: image: - productVersion: 1.25.0 + productVersion: 1.27.0 clusterConfig: authentication: - authenticationClass: nifi-admin-credentials @@ -52,7 +52,7 @@ kind: Secret metadata: name: nifi-admin-credentials-secret stringData: - admin: {{ nifiAdminPassword }} + admin: {{nifiAdminPassword}} --- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode From 7f4565b4abf6b8f270862897136aa6bae73c6399 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 18 Sep 2024 11:15:19 +0200 Subject: [PATCH 11/20] chore: Upgrade the Vector aggregator --- stacks/_templates/vector-aggregator.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml index cdedc2ee..91bbdbd6 100644 --- a/stacks/_templates/vector-aggregator.yaml +++ b/stacks/_templates/vector-aggregator.yaml @@ -3,11 +3,11 @@ name: vector repo: name: vector url: https://helm.vector.dev -version: 0.35.0 # app version 0.40.0 +version: 0.36.1 # app version 0.41.1 options: commonLabels: stackable.tech/vendor: Stackable - podLabels: # Doesn't seem to work? + podLabels: stackable.tech/vendor: Stackable role: Aggregator customConfig: @@ -25,7 +25,7 @@ options: - https://opensearch-cluster-master.default.svc.cluster.local:9200 mode: bulk # The auto-detection of the API version does not work in Vector - # 0.39.0 for OpenSearch, so the version must be set explicitly + # 0.41.1 for OpenSearch, so the version must be set explicitly # (see https://github.com/vectordotdev/vector/issues/17690). api_version: v8 tls: From 9b5f892141716570d468c60ee8f2e7fce818878b Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Fri, 27 Sep 2024 10:31:47 +0200 Subject: [PATCH 12/20] chore(druid): Bump to 30.0.0 --- stacks/keycloak-opa-poc/druid.yaml | 4 ++-- stacks/nifi-kafka-druid-superset-s3/druid.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks/keycloak-opa-poc/druid.yaml b/stacks/keycloak-opa-poc/druid.yaml index 514761ae..12cc47a4 100644 --- a/stacks/keycloak-opa-poc/druid.yaml +++ b/stacks/keycloak-opa-poc/druid.yaml @@ -5,7 +5,7 @@ metadata: name: druid spec: image: - productVersion: 28.0.1 + productVersion: 30.0.0 clusterConfig: listenerClass: external-unstable deepStorage: @@ -67,7 +67,7 @@ spec: "druid-pac4j"] # basic authenticator needed for internal authentication among Druid processes - # Trying to use the pac4j authenticator in the escalator below leads to 302 errors, + # Trying to use the pac4j authenticator in the escalator below leads to 302 errors, # it seems like the Druid processes cannot handle the OIDC authentication flow. druid.auth.authenticator.MyBasicMetadataAuthenticator.type: basic druid.auth.authenticator.MyBasicMetadataAuthenticator.initialInternalClientPassword: '${env:DRUID_SYSTEM_USER_PASSWORD}' # Default password for internal 'druid_system' user diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index 5407d200..dae6d10a 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -5,7 +5,7 @@ metadata: name: druid spec: image: - productVersion: 28.0.1 + productVersion: 30.0.0 clusterConfig: listenerClass: external-unstable zookeeperConfigMapName: druid-znode @@ -101,4 +101,4 @@ metadata: name: druid-db-credentials stringData: username: druid - password: druid \ No newline at end of file + password: druid From 25cb8a3abc722c7ce307c65111f62e4c21ff41ff Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Fri, 27 Sep 2024 10:38:27 +0200 Subject: [PATCH 13/20] chore(pre-commit): Ignore braces lint for templates yaml files --- .yamllint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamllint.yaml b/.yamllint.yaml index a944498c..ac4941c4 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -7,3 +7,4 @@ rules: check-keys: false comments: min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 + braces: disable # because the yaml files are templates which can have {{ ... }} From b2aa6613dec7732e867973095ee35571c0ed82cf Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Oct 2024 13:30:35 +0200 Subject: [PATCH 14/20] chore: lint --- stacks/airflow/airflow.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 5e8b96ce..c12394e9 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -1,4 +1,4 @@ -{% raw %} +# {% raw %} --- apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster @@ -285,7 +285,7 @@ data: limit: 512Mi driver: config: - resources: + resources: cpu: min: 1000m max: 1200m @@ -293,14 +293,14 @@ data: limit: 1024Mi executor: config: - resources: + resources: cpu: min: 500m max: 1000m memory: limit: 1024Mi replicas: 3 -{% endraw %} +# {% endraw %} --- apiVersion: v1 kind: Secret From 6dec8e59cb2b2715f729d7195ad9fab4f241ded7 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:45:34 +0100 Subject: [PATCH 15/20] Apply suggestions from code review Use images that exist. We can't use hdfs from 24.7 because there is a tool needed. For consistency, we use the same image for consistency --- demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml | 2 +- .../load-test-data.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml b/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml index 44300127..67cfe784 100644 --- a/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml +++ b/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml @@ -11,7 +11,7 @@ spec: # We use 24.3.0 here which contains the distcp MapReduce components # This is not included in the 24.7 images and will fail. # See: https://github.com/stackabletech/docker-images/issues/793 - image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.11.0 + image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev env: - name: HADOOP_USER_NAME value: stackable diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml index 7dd44f89..d02c5088 100644 --- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: load-ny-taxi-data - image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.11.0 + image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev # yamllint disable rule:line-length command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \ && cd /tmp \ From 46804991e518afba98b8153a62e22cebb14c734f Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:05:04 +0100 Subject: [PATCH 16/20] Apply suggestions from code review --- demos/signal-processing/Dockerfile-nifi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index 052cacd2..be1c2594 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,5 +1,5 @@ # docker build -f ./Dockerfile-nifi -t docker.stackable.tech/demos/nifi:1.27.0-postgresql . -FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.11.0 +FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.7.0 RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar" From ec104d513598849fe16b25d5eb2127e248e97f07 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 11 Nov 2024 14:07:42 +0100 Subject: [PATCH 17/20] chore: fix lint issue --- stacks/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index c12394e9..d382de3c 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -1,5 +1,5 @@ -# {% raw %} --- +# {% raw %} apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: From 7649d89639d2fd5e84bd4e8e3fa07a3df34b81fa Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:14:13 +0100 Subject: [PATCH 18/20] Apply suggestions from code review --- demos/signal-processing/Dockerfile-nifi | 2 -- stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index be1c2594..db643c3f 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,5 +1,3 @@ -# docker build -f ./Dockerfile-nifi -t docker.stackable.tech/demos/nifi:1.27.0-postgresql . - FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.7.0 RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar" diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml index c59001b7..de3f4d99 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml @@ -52,7 +52,7 @@ kind: Secret metadata: name: nifi-admin-credentials-secret stringData: - admin: {{nifiAdminPassword}} + admin: {{ nifiAdminPassword }} --- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode From cc89f6774071c7e3287f7e05a31bfe54cf9b373a Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 11 Nov 2024 14:16:09 +0100 Subject: [PATCH 19/20] chore: fix lint issue --- stacks/end-to-end-security/trino-regorules.yaml | 4 ++-- stacks/monitoring/grafana-dashboards.yaml | 5 +++-- stacks/signal-processing/grafana-dashboards.yaml | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/stacks/end-to-end-security/trino-regorules.yaml b/stacks/end-to-end-security/trino-regorules.yaml index fdd55d87..4a1afd9f 100644 --- a/stacks/end-to-end-security/trino-regorules.yaml +++ b/stacks/end-to-end-security/trino-regorules.yaml @@ -1,5 +1,5 @@ -{% raw %} --- +# {% raw %} apiVersion: v1 kind: ConfigMap metadata: @@ -1797,4 +1797,4 @@ data: regex.match(pattern_with_anchors, value) } -{% endraw %} +# {% endraw %} diff --git a/stacks/monitoring/grafana-dashboards.yaml b/stacks/monitoring/grafana-dashboards.yaml index 0df74960..523c2ed7 100644 --- a/stacks/monitoring/grafana-dashboards.yaml +++ b/stacks/monitoring/grafana-dashboards.yaml @@ -1,4 +1,5 @@ -{% raw %} +--- +# {% raw %} apiVersion: v1 kind: ConfigMap metadata: @@ -10457,4 +10458,4 @@ data: "version": 17, "weekStart": "" } -{% endraw %} +# {% endraw %} diff --git a/stacks/signal-processing/grafana-dashboards.yaml b/stacks/signal-processing/grafana-dashboards.yaml index 24ac2427..6ca6a2fc 100644 --- a/stacks/signal-processing/grafana-dashboards.yaml +++ b/stacks/signal-processing/grafana-dashboards.yaml @@ -1,4 +1,5 @@ -{% raw %} +--- +# {% raw %} apiVersion: v1 kind: ConfigMap metadata: @@ -2206,4 +2207,4 @@ data: "version": 7, "weekStart": "" } -{% endraw %} +# {% endraw %} From 97a3f5e397f58035455cf81c55a91c31d92a480d Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 11 Nov 2024 14:20:17 +0100 Subject: [PATCH 20/20] chore: disable the brace lint --- .yamllint.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.yamllint.yaml b/.yamllint.yaml index e08cc89c..ac4941c4 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -7,8 +7,4 @@ rules: check-keys: false comments: min-spaces-from-content: 1 # Needed due to https://github.com/adrienverge/yamllint/issues/443 - braces: - # because the yaml files are templates which can have {{ ... }} - ignore: | - stacks/signal-processing/* - stacks/_templates/* + braces: disable # because the yaml files are templates which can have {{ ... }}