docker-compose.yaml

# Docker Compose for Local Development
# Uses local storage alternatives (Azurite for Azure Storage emulation)
#
# Prerequisites:
#   1. Create the external network first:
#      docker compose -f docker-compose-network.yaml up -d
#   2. Copy .env.local to .env:
#      cp .env.local .env
#
# Usage:
#   docker compose up -d
#   OR with explicit env file:
#   docker compose --env-file .env.local up -d
#
# Services:
#   - dagster-webserver:  http://localhost:3001
#   - spark-master-ui:    http://localhost:8070
#   - spark-worker-1-ui:  http://localhost:8071
#   - spark-worker-2-ui:  http://localhost:8072
#   - hive-metastore:     thrift://localhost:9083
#   - azurite (blob):     http://localhost:10000
#   - azurite (queue):    http://localhost:10001
#   - azurite (table):    http://localhost:10002

x-common-config: &common-config
  init: true
  restart: unless-stopped

x-dagster-common-config: &dagster-common-config
  <<: *common-config
  env_file: .env
  build:
    context: ./dagster
    dockerfile: Dockerfile
  # Local build - no external registry needed
  environment:
    DAGSTER_HOME: ${DAGSTER_HOME}
    COMMIT_SHA: ${COMMIT_SHA:-local}
    DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS: ${DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS}
    # PostgreSQL connection
    PG_HOST: ${PG_HOST}
    PG_PORT: ${PG_PORT}
    # Hive Metastore connection
    HIVE_METASTORE_URI: ${HIVE_METASTORE_URI}
    # Spark connection
    SPARK_MASTER_URL: ${SPARK_MASTER_URL}
    # Local Azure Storage (Azurite) configuration
    AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
    AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
    AZURE_STORAGE_ACCOUNT_KEY: ${AZURE_STORAGE_ACCOUNT_KEY}
    AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
    USE_AZURITE: ${USE_AZURITE}
  volumes:
    - ./dagster:/app
    - ./dagster/.tmp/warehouse:/opt/spark/warehouse
    - ./dagster/.venv-docker:/app/.venv
  depends_on:
    dagster-storage:
      condition: service_healthy
    azurite:
      condition: service_healthy

x-spark-common-config: &spark-common-config
  <<: *common-config
  user: "1001:1001"
  # Custom Dockerfile based on bitnami/spark:3.5.0 with Azure/Delta dependencies
  build:
    context: .
    dockerfile: spark/Dockerfile
  env_file: .env
  environment:
    SPARK_RPC_AUTHENTICATION_ENABLED: ${SPARK_RPC_AUTHENTICATION_ENABLED}
    SPARK_RPC_ENCRYPTION_ENABLED: ${SPARK_RPC_ENCRYPTION_ENABLED}
    SPARK_RPC_AUTHENTICATION_SECRET: ${SPARK_RPC_AUTHENTICATION_SECRET}
    SPARK_SSL_NEED_CLIENT_AUTH: ${SPARK_SSL_NEED_CLIENT_AUTH}
    # Local Azure Storage (Azurite)
    AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
    AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
    AZURE_STORAGE_ACCOUNT_KEY: ${AZURE_STORAGE_ACCOUNT_KEY}
    AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
    USE_AZURITE: ${USE_AZURITE}

x-spark-worker-common-config: &spark-worker-common-config
  <<: *spark-common-config
  working_dir: /opt/bitnami/spark/app
  volumes:
    - ./dagster:/opt/bitnami/spark/app
  depends_on:
    - spark-master

x-spark-worker-common-env: &spark-worker-common-env
  SPARK_MODE: worker
  SPARK_MASTER_URL: ${SPARK_MASTER_URL}
  SPARK_WORKER_HOST: 0.0.0.0
  SPARK_WORKER_CORES: ${SPARK_WORKER_CORES}
  SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY}
  SPARK_PUBLIC_DNS: localhost
  PYTHONPATH: /opt/bitnami/spark/app

volumes:
  dagster-storage:
  hms-db-storage:
  azurite-data:
  spark-warehouse:

networks:
  default:
    name: giga-dataops
    external: true

services:
  # =============================================================================
  # LOCAL STORAGE (Azurite - Azure Storage Emulator)
  # =============================================================================

  azurite:
    <<: *common-config
    image: mcr.microsoft.com/azure-storage/azurite:3.29.0
    hostname: ${AZURE_STORAGE_ACCOUNT_NAME}.blob.core.windows.net
    # Use port 80 for blob service so wasb:// (HTTP) protocol works with Hadoop
    command: "azurite --blobHost 0.0.0.0 --blobPort 80 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --loose --skipApiVersionCheck"
    volumes:
      - azurite-data:/data
    ports:
      - "10000:80"    # Blob service (internal 80 for wasb://, exposed as 10000)
      - "10001:10001"  # Queue service
      - "10002:10002"  # Table service
    healthcheck:
      test: ["CMD", "nc", "-z", "127.0.0.1", "80"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      default:
        aliases:
          - ${AZURE_STORAGE_ACCOUNT_NAME}.blob.core.windows.net

  # Initialize Azurite container on startup
  azurite-init:
    image: mcr.microsoft.com/azure-cli:2.57.0
    depends_on:
      azurite:
        condition: service_healthy
    environment:
      AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
      AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
      AZURE_DELTA_LAKE_CONTAINER_NAME: ${AZURE_DELTA_LAKE_CONTAINER_NAME}
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        echo "Creating blob container..."
        az storage container create --name $$AZURE_BLOB_CONTAINER_NAME --connection-string "$$AZURE_STORAGE_CONNECTION_STRING" || true
        az storage container create --name $$AZURE_DELTA_LAKE_CONTAINER_NAME --connection-string "$$AZURE_STORAGE_CONNECTION_STRING" || true
        echo "Blob containers created successfully"
    restart: "no"

  # =============================================================================
  # DAGSTER SERVICES
  # =============================================================================

  dagster-storage:
    <<: *common-config
    image: public.ecr.aws/bitnami/postgresql:14.10.0-debian-11-r31
    env_file: .env
    environment:
      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
      POSTGRESQL_PGAUDIT_LOG: "NONE"
    volumes:
      - dagster-storage:/bitnami/postgresql
    ports:
      - "5433:5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  dagster-webserver:
    <<: *dagster-common-config
    command: ["./docker-entrypoint.dev.sh"]
    ports:
      - "3001:3002"
      - "5678:5678"  # Debug port
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3002/server_info"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

  dagster-daemon:
    <<: *dagster-common-config
    command: ["bash", "-c", "poetry install --with dev,dagster,pipelines,spark && poetry run dagster-daemon run"]
    environment:
      DAGSTER_HOME: ${DAGSTER_HOME}
      COMMIT_SHA: ${COMMIT_SHA:-local}
      DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS: ${DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS}
      PG_HOST: ${PG_HOST}
      PG_PORT: ${PG_PORT}
      HIVE_METASTORE_URI: ${HIVE_METASTORE_URI}
      SPARK_MASTER_URL: ${SPARK_MASTER_URL}
      DAGSTER_DAEMON_HEARTBEAT_TOLERANCE: ${DAGSTER_DAEMON_HEARTBEAT_TOLERANCE}
      # Local Azure Storage (Azurite)
      AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
      AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
      AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}

  # =============================================================================
  # SPARK SERVICES
  # =============================================================================

  spark-master:
    <<: *spark-common-config
    environment:
      SPARK_MODE: master
      SPARK_MASTER_HOST: 0.0.0.0
      SPARK_MASTER_PORT: 7077
      SPARK_MASTER_WEBUI_PORT: 8070
      SPARK_PUBLIC_DNS: localhost
      THRIFT_PORT: 10000
      SPARK_RPC_AUTHENTICATION_ENABLED: ${SPARK_RPC_AUTHENTICATION_ENABLED}
      SPARK_RPC_ENCRYPTION_ENABLED: ${SPARK_RPC_ENCRYPTION_ENABLED}
      SPARK_RPC_AUTHENTICATION_SECRET: ${SPARK_RPC_AUTHENTICATION_SECRET}
      # Local Azure Storage (Azurite)
      AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
      AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
      AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
    volumes:
      - spark-warehouse:/opt/spark/warehouse
    ports:
      - "7077:7077"   # Spark master
      - "8070:8070"   # Spark master UI
      - "4040:4040"   # Spark driver UI
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8070"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    depends_on:
      azurite:
        condition: service_healthy

  spark-worker-1:
    <<: [*spark-common-config, *spark-worker-common-config]
    environment:
      <<: *spark-worker-common-env
      SPARK_WORKER_WEBUI_PORT: 8071
      SPARK_RPC_AUTHENTICATION_ENABLED: ${SPARK_RPC_AUTHENTICATION_ENABLED}
      SPARK_RPC_ENCRYPTION_ENABLED: ${SPARK_RPC_ENCRYPTION_ENABLED}
      SPARK_RPC_AUTHENTICATION_SECRET: ${SPARK_RPC_AUTHENTICATION_SECRET}
      # Local Azure Storage (Azurite)
      AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
      AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
      AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
    ports:
      - "8071:8071"

  spark-worker-2:
    <<: [*spark-common-config, *spark-worker-common-config]
    environment:
      <<: *spark-worker-common-env
      SPARK_WORKER_WEBUI_PORT: 8072
      SPARK_RPC_AUTHENTICATION_ENABLED: ${SPARK_RPC_AUTHENTICATION_ENABLED}
      SPARK_RPC_ENCRYPTION_ENABLED: ${SPARK_RPC_ENCRYPTION_ENABLED}
      SPARK_RPC_AUTHENTICATION_SECRET: ${SPARK_RPC_AUTHENTICATION_SECRET}
      # Local Azure Storage (Azurite)
      AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
      AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME}
      AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
    ports:
      - "8072:8072"

  # =============================================================================
  # HIVE METASTORE SERVICES
  # =============================================================================

  hms-database:
    <<: *common-config
    image: public.ecr.aws/bitnami/postgresql:14.10.0-debian-11-r31
    env_file: .env
    environment:
      POSTGRESQL_USERNAME: ${HMS_POSTGRESQL_USERNAME}
      POSTGRESQL_PASSWORD: ${HMS_POSTGRESQL_PASSWORD}
      POSTGRESQL_DATABASE: ${HMS_POSTGRESQL_DATABASE}
      POSTGRESQL_PGAUDIT_LOG: "NONE"
      POSTGRESQL_INITSCRIPTS_USERNAME: ${HMS_POSTGRESQL_USERNAME}
      POSTGRESQL_INITSCRIPTS_PASSWORD: ${HMS_POSTGRESQL_PASSWORD}
      POSTGRESQL_PG_HBA_CONF: |
        # TYPE  DATABASE        USER            ADDRESS                 METHOD
        local   all            all                                     trust
        host    all            all             127.0.0.1/32            md5
        host    all            all             ::1/128                 md5
        host    all            all             0.0.0.0/0               md5
    volumes:
      - hms-db-storage:/bitnami/postgresql
    ports:
      - "5434:5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "${HMS_POSTGRESQL_USERNAME}"]
      timeout: 3s
      start_period: 5s
      retries: 5

  hive-metastore:
    <<: *common-config
    build:
      context: ./hive
      dockerfile: Dockerfile
    env_file: .env
    environment:
      DB_DRIVER: postgres
      SERVICE_NAME: metastore
      METASTORE_HOME: /opt/hive
      HMS_DATABASE_URL: jdbc:postgresql://hms-database:5432/${HMS_POSTGRESQL_DATABASE}
      HMS_POSTGRESQL_USERNAME: ${HMS_POSTGRESQL_USERNAME}
      HMS_POSTGRESQL_PASSWORD: ${HMS_POSTGRESQL_PASSWORD}
      # Local development - use local file path for warehouse (simpler than wasbs with Azurite)
      STORAGE_ACCOUNT_NAME: ${STORAGE_ACCOUNT_NAME}
      STORAGE_CONTAINER_NAME: ${STORAGE_CONTAINER_NAME}
      AZURE_STORAGE_ACCOUNT_KEY: ${AZURE_STORAGE_ACCOUNT_KEY}
      AZURE_SAS_TOKEN: ${AZURE_SAS_TOKEN}
      # Use wasb:// (HTTP) for warehouse - wasbs:// (HTTPS) times out with Azurite
      METASTORE_WAREHOUSE_DIR: ${METASTORE_WAREHOUSE_DIR}
    volumes:
      - ./hive/hive-site.template.xml:/opt/hive/tpl/hive-site.template.xml
      - ./hive/metastore-site.local.template.xml:/opt/hive/tpl/metastore-site.template.xml
      - ./hive/hms-entrypoint.sh:/opt/hive/bin/hms-entrypoint.sh
      - spark-warehouse:/opt/spark/warehouse
    ports:
      - "9083:9083"
    depends_on:
      hms-database:
        condition: service_healthy
      azurite:
        condition: service_healthy