docker-compose.yml

name: de-bootcamp

volumes:
  pg_data:
  minio_data:
  trino_data:
  hive_metastore_db_data:
  metabase_data:
  kafka_data:
  superset_data:
  airflow_db_data:
  redis_data:

services:
  ###########################################################################
  # CORE: PostgreSQL (app db)
  ###########################################################################
  postgres:
    image: postgres:15
    container_name: de_postgres
    restart: unless-stopped
    command: ["postgres", "-c", "wal_level=logical"]
    environment:
      POSTGRES_USER: de_user
      POSTGRES_PASSWORD: de_pass
      POSTGRES_DB: de_db
    ports:
      - "5432:5432"
    volumes:
      - pg_data:/var/lib/postgresql/data

  ###########################################################################
  # DATA LAKEHOUSE: MinIO (S3)
  ###########################################################################
  minio:
    image: quay.io/minio/minio:RELEASE.2024-05-01T01-11-10Z
    container_name: de_minio
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
    ports:
      - "9000:9000"  # S3 API
      - "9001:9001"  # Console
    volumes:
      - minio_data:/data

  minio-create-bucket:
    image: minio/mc
    container_name: de_minio_create_bucket
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      mc alias set local http://minio:9000 minioadmin minioadmin &&
      mc mb -p local/lakehouse || true &&
      mc mb -p local/raw || true &&
      mc mb -p local/bronze || true &&
      mc mb -p local/silver || true &&
      mc mb -p local/gold || true &&
      exit 0
      "
    restart: "no"

  ###########################################################################
  # HIVE METASTORE: Postgres DB + Metastore service
  ###########################################################################
  hive-metastore-db:
    image: postgres:15
    environment:
      POSTGRES_DB: metastore
      POSTGRES_USER: hive
      POSTGRES_PASSWORD: hive
      POSTGRES_INITDB_ARGS: "--auth-host=md5 --auth-local=trust"

    ports:
      - "5433:5432"
    volumes:
      - hive_metastore_db_data:/var/lib/postgresql/data

    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U hive -d metastore"]
      interval: 5s
      timeout: 5s
      retries: 20

  hive-metastore:
    image: apache/hive:3.1.3
    platform: linux/amd64
    depends_on:
      hive-metastore-db:
        condition: service_healthy
    environment:
      SERVICE_NAME: metastore
      # Entrypoint sets SKIP_SCHEMA_INIT from IS_RESUME. false = run schematool -initSchema once.
      # After first successful start, set to "true" and restart so restarts don't re-init (else: "BUCKETING_COLS already exists").
      IS_RESUME: "true"
      DB_DRIVER: postgres
      SERVICE_OPTS: >
        -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://hive-metastore-db:5432/metastore
        -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
        -Djavax.jdo.option.ConnectionUserName=hive
        -Djavax.jdo.option.ConnectionPassword=hive
    ports:
      - "9083:9083"
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo >/dev/tcp/127.0.0.1/9083' || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 30s

  ###########################################################################
  # TRINO
  # NOTE:
  # - trino config must exist at ./trino/etc (node.properties, config.properties, jvm.config, log.properties, catalog/)
  # - data dir mapped to /data/trino to avoid permission issues
  ###########################################################################
  trino:
    image: trinodb/trino:435
    container_name: de_trino
    restart: unless-stopped
    user: "0"
    ports:
      - "8080:8080"
    depends_on:
      hive-metastore:
        condition: service_healthy
      minio:
        condition: service_started
    volumes:
      - ./trino/etc:/etc/trino:ro
      - trino_data:/data/trino

  ###########################################################################
  # BI: Metabase (Week 4 Buổi 7 — Dashboard)
  # UI: http://localhost:3000 — Add Postgres: host=postgres, port=5432, db=de_db
  ###########################################################################
  metabase:
    image: metabase/metabase:v0.49.0
    container_name: de_metabase
    restart: unless-stopped
    ports:
      - "3000:3000"
    depends_on:
      - postgres
    environment:
      MB_JETTY_HOST: "0.0.0.0"
    volumes:
      - metabase_data:/data

  ###########################################################################
  # STREAMING: Zookeeper + Kafka (Week 6 Buổi 12 — Kafka Fundamentals)
  # Start: docker compose --profile streaming up -d
  # Bootstrap (host): localhost:9092
  ###########################################################################
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    container_name: de_zookeeper
    restart: unless-stopped
    profiles:
      - streaming
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - "2181:2181"

  kafka:
    image: confluentinc/cp-kafka:7.5.0
    container_name: de_kafka
    restart: unless-stopped
    profiles:
      - streaming
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
    volumes:
      - kafka_data:/var/lib/kafka/data

  ###########################################################################
  # CDC: Kafka Connect + Debezium (Week 7 Buổi 13 — CDC PostgreSQL → Kafka)
  # REST API: http://localhost:8083
  # Start: docker compose --profile streaming up -d (after Kafka + Zookeeper)
  ###########################################################################
  connect:
    image: debezium/connect:2.4
    container_name: de_connect
    restart: unless-stopped
    profiles:
      - streaming
    depends_on:
      - kafka
    ports:
      - "8083:8083"
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: connect-cluster
      CONFIG_STORAGE_TOPIC: connect-configs
      OFFSET_STORAGE_TOPIC: connect-offsets
      STATUS_STORAGE_TOPIC: connect-status
      CONFIG_STORAGE_REPLICATION_FACTOR: 1
      OFFSET_STORAGE_REPLICATION_FACTOR: 1
      STATUS_STORAGE_REPLICATION_FACTOR: 1
      KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONFIG_STORAGE_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      OFFSET_STORAGE_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      OFFSET_STORAGE_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter

  ###########################################################################
  # STREAMING: Flink (Week 7 Buổi 14 — Kafka → Flink → MinIO)
  # Web UI: http://localhost:8081
  # Cần thêm JAR Kafka connector vào /opt/flink/lib (xem scripts/download_flink_connectors.sh)
  ###########################################################################
  flink-jobmanager:
    image: apache/flink:1.18.0-scala_2.12-java11
    container_name: de_flink_jobmanager
    restart: unless-stopped
    profiles:
      - streaming
    depends_on:
      - kafka
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - FLINK_PROPERTIES=jobmanager.rpc.address=flink-jobmanager
      - AWS_ACCESS_KEY_ID=minioadmin
      - AWS_SECRET_ACCESS_KEY=minioadmin

  flink-taskmanager:
    image: apache/flink:1.18.0-scala_2.12-java11
    container_name: de_flink_taskmanager
    restart: unless-stopped
    profiles:
      - streaming
    depends_on:
      - flink-jobmanager
    command: taskmanager
    environment:
      - FLINK_PROPERTIES=jobmanager.rpc.address=flink-jobmanager
      - AWS_ACCESS_KEY_ID=minioadmin
      - AWS_SECRET_ACCESS_KEY=minioadmin

  ###########################################################################
  # BI: Superset (Week 7 Buổi 14 — Realtime dashboard)
  # UI: http://localhost:8088 — admin / admin
  ###########################################################################
  superset:
    image: apache/superset:3.0.0
    container_name: de_superset
    restart: unless-stopped
    profiles:
      - streaming
    depends_on:
      - postgres
    ports:
      - "8088:8088"
    environment:
      SUPERSET_SECRET_KEY: "dev-secret-key-change-in-production"
      SUPERSET_LOAD_EXAMPLES: "no"
    volumes:
      - superset_data:/app/superset_home
    user: "root"
    command: >
      /bin/sh -c "
      superset db upgrade &&
      (superset fab create-admin --username admin --firstname Admin --lastname User --email admin@localhost --password admin 2>/dev/null || true) &&
      superset init &&
      superset run -h 0.0.0.0 -p 8088
      "

  ###########################################################################
  # ORCHESTRATION: Airflow (Week 8 Buổi 15 — DAG ingest → clean → load → validate)
  # Start: docker compose --profile airflow up -d (sau khi chạy airflow-init một lần)
  # UI: http://localhost:8082 — airflow / airflow
  ###########################################################################
  airflow-db:
    image: postgres:15
    container_name: de_airflow_db
    restart: unless-stopped
    profiles:
      - airflow
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    ports:
      - "5434:5432"
    volumes:
      - airflow_db_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
      interval: 5s
      timeout: 5s
      retries: 10

  airflow-init:
    image: apache/airflow:2.8.0
    container_name: de_airflow_init
    profiles:
      - airflow
    depends_on:
      airflow-db:
        condition: service_healthy
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-db:5432/airflow
      _AIRFLOW_WWW_USER_USERNAME: airflow
      _AIRFLOW_WWW_USER_PASSWORD: airflow
      AIRFLOW_UID: "0"
    command: >
      bash -c "
      airflow db init &&
      airflow users create --username airflow --firstname Admin --lastname User --role Admin --email admin@localhost --password airflow 2>/dev/null || true &&
      echo 'Airflow DB initialized.'
      "
    user: "root"

  airflow-webserver:
    image: apache/airflow:2.8.0
    container_name: de_airflow_webserver
    restart: unless-stopped
    profiles:
      - airflow
    depends_on:
      airflow-db:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-db:5432/airflow
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "true"
      AIRFLOW_UID: "0"
    ports:
      - "8082:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
    command: webserver
    user: "root"

  airflow-scheduler:
    image: apache/airflow:2.8.0
    container_name: de_airflow_scheduler
    restart: unless-stopped
    profiles:
      - airflow
    depends_on:
      airflow-db:
        condition: service_healthy
      airflow-init:
        condition: service_completed_successfully
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-db:5432/airflow
      AIRFLOW_UID: "0"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
    command: scheduler
    user: "root"

  ###########################################################################
  # FEATURE STORE: Redis (Week 9 Buổi 17 — Feast online store)
  # Start: docker compose --profile featurestore up -d
  # Connection: redis:6379 (trong Docker), localhost:6379 (từ host)
  ###########################################################################
  redis:
    image: redis:7-alpine
    container_name: de_redis
    restart: unless-stopped
    profiles:
      - featurestore
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data