From 1e0a2feaf3ebdfeaacbbdbaf78b9a83dcf39bf49 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 19 Mar 2025 15:12:58 +0200 Subject: [PATCH 1/2] fix: better encapsulation for the anomaly detection stack --- .../s3-connection.yaml | 32 +++++++++++++++++++ stacks/stacks-v2.yaml | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 stacks/spark-trino-superset-s3/s3-connection.yaml diff --git a/stacks/spark-trino-superset-s3/s3-connection.yaml b/stacks/spark-trino-superset-s3/s3-connection.yaml new file mode 100644 index 00000000..55ae8400 --- /dev/null +++ b/stacks/spark-trino-superset-s3/s3-connection.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Connection +metadata: + name: minio +spec: + # Hostname must match secret op's certificate SAN + host: minio.default.svc.cluster.local + port: 9000 + accessStyle: Path + credentials: + secretClass: minio-s3-credentials +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: minio-s3-credentials +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-s3-credentials + labels: + secrets.stackable.tech/class: minio-s3-credentials +stringData: + accessKey: admin + secretKey: {{ minioAdminPassword }} diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b4bf0069..a19b9356 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -258,7 +258,7 @@ stacks: - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/s3-connection.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/s3-connection.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/hive-metastore.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/trino.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml From 898f1296abe845e20dca44c20d442e6322215a87 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 19 Mar 2025 15:19:36 +0200 Subject: [PATCH 2/2] remove unused dependency --- .../create-spark-anomaly-detection-job.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 3ca2b3f1..3623b9e7 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -43,7 +43,6 @@ data: deps: packages: - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 - - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.5 requirements: - scikit-learn==1.4.0 s3connection: