Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 48 additions & 1 deletion demos/trino-taxi-data/load-test-data.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,60 @@ spec:
- name: load-ny-taxi-data
image: "bitnami/minio:2024-debian-12"
# yamllint disable-line rule:line-length
command: ["bash", "-c", "cd /tmp && for month in 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp yellow_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/; done"]
command:
- bash
- -ce
- |
# Copy the CA cert from the "tls" SecretClass
cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt
MINIO_ENDPOINT="https://minio.default.svc.cluster.local:9000/"
MINIO_ACCESS_KEY=$(cat /minio-s3-credentials/accessKey)
MINIO_SECRET_KEY=$(cat /minio-s3-credentials/secretKey)
cd /tmp
for month in \
2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 \
2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 \
2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do
curl -O "https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet"
mc alias set minio "$MINIO_ENDPOINT" "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY"
mc cp "yellow_tripdata_$month.parquet" minio/demo/ny-taxi-data/raw/
done
volumeMounts:
- name: minio-s3-credentials
mountPath: /minio-s3-credentials
# Mount the certificate generated by the secret-operator
- name: tls
mountPath: /etc/minio/mc/original_certs
# On startup, we will rename the certs and move them here:
- mountPath: /.mc/certs/CAs
name: certs

volumes:
- name: minio-s3-credentials
secret:
secretName: minio-s3-credentials
# Request a TLS certificate from the secret-operator
- name: tls
ephemeral:
volumeClaimTemplate:
metadata:
annotations:
secrets.stackable.tech/class: tls
secrets.stackable.tech/scope: |-
service=minio
spec:
storageClassName: secrets.stackable.tech
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1
# Create an in-memory emptyDir to copy the certs to (to avoid permission errors)
- name: certs
emptyDir:
sizeLimit: 5Mi
medium: Memory
restartPolicy: OnFailure
backoffLimit: 50