docker-compose.yml

include:
  - docker-compose-airflow.yml

x-common-environment: &common-environment
  AWS_ACCESS_KEY_ID: admin
  AWS_SECRET_ACCESS_KEY: password
  AWS_REGION: us-east-1

services:
# Redpanda was changed to Confluent cloud platform
# Redpanda cluster
#  redpanda-1:
#    image: redpandadata/redpanda:v24.3.8
#    container_name: redpanda-1
#    command:
#      - redpanda
#      - start
#      - --smp
#      - '1'
#      - --reserve-memory
#      - 0M
#      - --overprovisioned
#      - --node-id
#      - '1'
#      - --kafka-addr
#      - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
#      - --advertise-kafka-addr
#      - PLAINTEXT://redpanda-1:29092,OUTSIDE://localhost:9092
#      - --pandaproxy-addr
#      - PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082
#      - --advertise-pandaproxy-addr
#      - PLAINTEXT://redpanda-1:28082,OUTSIDE://localhost:8082
#      - --rpc-addr
#      - 0.0.0.0:33145
#      - --advertise-rpc-addr
#      - redpanda-1:33145
#    ports:
#      # - 8081:8081
#      - 8082:8082
#      - 9092:9092
#      - 9644:9644
#      - 28082:28082
#      - 29092:29092
#    healthcheck:
#      test: ["CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true' || exit 1"]
#      interval: 10s
#      timeout: 5s
#      retries: 5
#      start_period: 30s
#
#  redpanda-console:
#    image: redpandadata/console:v2.8.4
#    container_name: redpanda-console
#    entrypoint: /bin/sh
#    command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
#    environment:
#      CONFIG_FILEPATH: /tmp/config.yml
#      CONSOLE_CONFIG_FILE: |
#        kafka:
#          brokers: ["redpanda-1:29092"]
#          schemaRegistry:
#            enabled: false
#        redpanda:
#          adminApi:
#            enabled: true
#            urls: ["http://redpanda-1:9644"]
#        connect:
#          enabled: false
#    ports:
#      - 8084:8080
#    depends_on:
#      - redpanda-1
#
#
## Create topics
#  redpanda-init:
#    image: redpandadata/redpanda:v24.3.8
#    container_name: redpanda-init
#    depends_on:
#      - redpanda-console
#    entrypoint: ["/bin/bash","-c"]
#    environment:
#      RPK_BROKERS: redpanda-1:29092
#    command: |
#        "
#        rpk topic create raw_ticks
#        for i in m1 m5 m10 m15 m30 h1 h3 h4 D1 W1 M1; do echo "$$i"; rpk topic create "candle_$$i"; done;
#        "

# Mock S3 service using MinIO
  minio:
    image: minio/minio:RELEASE.2025-04-08T15-41-24Z
    container_name: minio
#    networks:
#      iceberg_net:
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
#      MINIO_DOMAIN: minio
    volumes:
      - ./minio-data:/data
    command: server /data --console-address ":9001"

  minio-mc:
    image: minio/mc:latest
#    networks:
#      iceberg_net:
    depends_on:
      - minio
    environment:
      <<: *common-environment
    entrypoint: ["/bin/sh", "-c"]
    command: |
      "
      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb minio/stocks-warehouse;
      /usr/bin/mc policy set public minio/stocks-warehouse;
      tail -f /dev/null
      "

  # REST catalog for Iceberg
  iceberg-rest:
    image: tabulario/iceberg-rest:1.6.0
    container_name: rest
#    networks:
#      iceberg_net:
    depends_on:
      - minio
    ports:
      - "8181:8181"
    environment:
      <<: *common-environment
      CATALOG_WAREHOUSE: s3://stocks-warehouse/
      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
      CATALOG_S3_ENDPOINT: http://minio:9000
      CATALOG_S3__ENDPOINT: http://minio:9000
      CATALOG_S3_PATH__STYLE__ACCESS: true
      AWS_S3_PATH_STYLE_ACCESS: true
#      CATALOG_URI: jdbc:postgresql://postgres/demo_catalog
#      CATALOG_JDBC_USER: admin
#      CATALOG_JDBC_PASSWORD: password


  trino-coordinator:
    image: trinodb/trino:474
    container_name: trino-coordinator
#    networks:
#      iceberg_net:
    environment:
      <<: *common-environment
      TRINO_CONF_PATH: /etc/trino
    ports:
      - "8090:8080"
    deploy:
      resources:
        limits:
          memory: 4G
        reservations:
          memory: 2G
    volumes:
      - ./trino/etc:/etc/trino:rw
#    environment:
#      - JAVA_HOME=/usr/lib/jvm/zulu11
#    command: ["bash", "-c", "java -version && /usr/lib/trino/bin/run-trino"]
    depends_on:
#      - redpanda-1
      - minio
      - iceberg-rest

  # Prometheus for metrics collection
  prometheus:
    image: prom/prometheus:v2.45.0
    container_name: prometheus
#    networks:
#      iceberg_net:
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--web.enable-lifecycle'
    depends_on:
      - trino-coordinator

  # Grafana for visualization
  grafana:
    image: grafana/grafana:10.2.0
    container_name: grafana
#    networks:
#      iceberg_net:
    ports:
      - "3000:3000"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - ./grafana/dashboards:/var/lib/grafana/dashboards
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=trino-datasource
    depends_on:
      - prometheus

  spark:
    image: tabulario/spark-iceberg:latest
    container_name: spark
#    networks:
#      iceberg_net:
    ports:
      - 8888:8888
      - 8083:8080
      - 10000:10000
      - 10001:10001
    depends_on:
      - iceberg-rest
      - minio
    environment:
      <<: *common-environment
    stdin_open: true
    tty: true
    command: >
      spark-sql
          --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
          --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog
          --conf spark.sql.catalog.demo.type=rest
          --conf spark.sql.catalog.demo.uri=http://rest:8181
          --conf spark.sql.catalog.demo.io-impl=org.apache.iceberg.aws.s3.S3FileIO
          --conf spark.sql.catalog.demo.warehouse=s3://stocks-warehouse/
          --conf spark.sql.catalog.demo.s3.endpoint=http://minio:9000
          --conf spark.sql.defaultCatalog=demo
          --conf spark.eventLog.enabled=true
          --conf spark.eventLog.dir=/home/iceberg/spark-events
          --conf spark.history.fs.logDirectory=/home/iceberg/spark-events
          --conf spark.sql.catalogImplementation=in-memory

volumes:
  minio-data:
  prometheus_data:
  grafana_data:

#networks:
#  iceberg_net:

#networks:
#  default:
#    name: trino-network
##    external: true