docker-compose.yml

version: "3.9"

networks:
  mynetwork:
    driver: bridge

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.2
    container_name: zookeeper
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
    ports:
      - "2181:2181"
    networks:
      - mynetwork
    healthcheck:
      test: echo ruok | nc localhost 2181
      interval: 10s
      timeout: 5s
      retries: 5

  kafka:
    image: confluentinc/cp-kafka:7.5.2
    container_name: kafka
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
    networks:
      - mynetwork
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      # 단일 브로커 안정화 값
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      # Retension 설정 넣기
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      KAFKA_LOG_RETENTION_HOURS: 24
      KAFKA_LOG_RETENTION_BYTES: 1073741824
      KAFKA_LOG_CLEANUP_POLICY: delete
      KAFKA_LOG_SEGMENT_BYTES: 134217728
      KAFKA_LOG_CLEANER_ENABLE: "true"
    command:
      - bash
      - -c
      - |
        /etc/confluent/docker/run &
        sleep 10
        kafka-topics --bootstrap-server localhost:29092 \
          --create --if-not-exists --topic webtoon_user_events_v2 \
          --partitions 12 --replication-factor 1
        wait
    healthcheck:
      test: ["CMD-SHELL", "kafka-topics --bootstrap-server localhost:29092 --list || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 5

  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    container_name: kafdrop
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "9000:9000"
    networks:
      - mynetwork
    restart: always
    environment:
      KAFKA_BROKERCONNECT: "kafka:29092"
      SERVER_PORT: 9000
      SERVER_ADDRESS: 0.0.0.0

  spark-master:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: spark-master
    hostname: spark-master
    env_file:
      - .env
    environment:
      - HOME=/opt/spark
      - USER=spark                            # 단순 리눅스 환경변수 (실제 리눅스 계정, 컨테이너 실행 권한과 관련 X)
      - HADOOP_USER_NAME=spark                # 내부적으로 HDFS API 호출 시, 권한 체크하는 Hadoop 계정
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
    ports:
      - "8080:8080"     # Web UI
      - "7077:7077"     # spark 클러스터 간 통신
    networks:
      - mynetwork
    volumes:
      - ./src/spark:/opt/workspace/src/spark
      - .env:/opt/workspace/.env
      - spark_checkpoints:/opt/workspace/checkpoints
      - spark_data:/opt/workspace/data
    command: >
      bash -c "/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master --host spark-master --port 7077 --webui-port 8080"
    deploy:
      resources:
        limits:
          cpus: "0.5"
          memory: 1G
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080"]
      interval: 10s
      timeout: 5s
      retries: 5

  spark-worker:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: spark-worker
    hostname: spark-worker
    depends_on:
      spark-master:
        condition: service_healthy
    env_file:
      - .env
    environment:
      - HOME=/opt/spark
      - USER=spark
      - HADOOP_USER_NAME=spark
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
    ports:
      - "8081:8081"
    networks:
      - mynetwork
    volumes:
      - ./src/spark:/opt/workspace/src/spark
      - .env:/opt/workspace/.env
      - spark_checkpoints:/opt/workspace/checkpoints
      - spark_data:/opt/workspace/data
    command: >
      bash -c "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --host spark-worker --webui-port 8081 --cores 2 --memory 16G spark://spark-master:7077"
    deploy:
      resources:
        limits:
          cpus: "2"
          memory: 16G
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081"]
      interval: 10s
      timeout: 5s
      retries: 5

  spark-worker-2:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: spark-worker-2
    hostname: spark-worker-2
    depends_on:
      spark-master:
        condition: service_healthy
    env_file:
      - .env
    environment:
      - HOME=/opt/spark
      - USER=spark
      - HADOOP_USER_NAME=spark
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
    ports:
      - "8082:8081"
    networks:
      - mynetwork
    volumes:
      - ./src/spark:/opt/workspace/src/spark
      - .env:/opt/workspace/.env
      - spark_checkpoints:/opt/workspace/checkpoints
      - spark_data:/opt/workspace/data
    command: >
      bash -c "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --host spark-worker-2 --webui-port 8081 --cores 2 --memory 8G spark://spark-master:7077"
    deploy:
      resources:
        limits:
          cpus: "2"
          memory: 8G
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081"]
      interval: 10s
      timeout: 5s
      retries: 5

  spark-worker-3:         # gold task 전용
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: spark-worker-3
    hostname: spark-worker-3
    depends_on:
      spark-master:
        condition: service_healthy
    env_file:
      - .env
    environment:
      - HOME=/opt/spark
      - USER=spark
      - HADOOP_USER_NAME=spark
      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
    ports:
      - "8083:8081"
    networks:
      - mynetwork
    volumes:
      - ./src/spark:/opt/workspace/src/spark
      - .env:/opt/workspace/.env
      - spark_checkpoints:/opt/workspace/checkpoints
      - spark_data:/opt/workspace/data
    command: >
        bash -c "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --host spark-worker-3 --webui-port 8081 --cores 2 --memory 16G spark://spark-master:7077"
    deploy:
      resources:
        limits:
          cpus: "2"
          memory: "16G"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081"]
      interval: 10s
      timeout: 5s
      retries: 5

  x-airflow-common: &airflow-common
    build:
      context: ./docker/airflow
      dockerfile: Dockerfile
    env_file:
      - .env
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
      AIRFLOW__WEBSERVER__WEB_SERVER_BASE_URL: "${AIRFLOW__WEBSERVER__WEB_BASE_URL}"
      AIRFLOW__CORE__DEFAULT_TIMEZONE: Asia/Seoul
      TZ: Asia/Seoul
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW_UID: ${AIRFLOW_UID}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_DEFAULT_REGION: ap-northeast-2
      # For spark submit
      SPARK_MASTER: ${SPARK_MASTER}
      SPARK_CHECKPOINT_DIR: ${SPARK_CHECKPOINT_DIR}
    networks:
      - mynetwork
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - airflow_logs:/opt/airflow/logs
      - ./src/spark:/opt/workspace/src/spark
      - airflow_plugins:/opt/airflow/plugins

  airflow-init:
    <<: *airflow-common
    container_name: airflow-init
    entrypoint: /bin/bash
    command:
      - -c
      - |
        airflow db upgrade && \
        airflow users create \
          --username byeolong2 \
          --firstname Hanbyeol \
          --lastname Kim \
          --role Admin \
          --email hbstella92@gmail.com \
          --password adminadmin12
    depends_on:
      postgres:
        condition: service_healthy

  airflow-webserver:
    <<: *airflow-common
    container_name: airflow-webserver
    depends_on:
      - postgres
      - airflow-init
    ports:
      - "8088:8080"
    restart: always
    command: webserver

  airflow-scheduler:
    <<: *airflow-common
    container_name: airflow-scheduler
    depends_on:
      - postgres
      - airflow-init
    restart: always
    command: scheduler

  postgres:
    image: postgres:14
    container_name: postgres
    env_file:
      - .env
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    ports:
      - "5432:5432"
    networks:
      - mynetwork
    volumes:
      - pg_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      retries: 5
      interval: 10s
      timeout: 5s

  trino:
    image: trinodb/trino:430
    container_name: trino
    ports:
      - "8085:8080"
    volumes:
      - ./trino/etc:/etc/trino
    env_file:
      - .env
    environment:
      JAVA_TOOL_OPTIONS: -Xmx4G
    depends_on:
      - iceberg-rest
    networks:
      - mynetwork
    restart: always

  iceberg-rest:
    image: tabulario/iceberg-rest:latest
    container_name: iceberg-rest
    ports:
      - "8181:8181"
    environment:
      CATALOG_WAREHOUSE: ${SPARK_PARQUET_WAREHOUSE}
      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO

      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_REGION: ${AWS_REGION}

      CATALOG_S3_ENDPOINT: https://s3.ap-northeast-2.amazonaws.com
      CATALOG_S3_REGION: ${AWS_REGION}
      CATALOG_S3_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      CATALOG_S3_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
    networks:
      - mynetwork
    restart: always

  grafana:
    build:
      context: ./docker/grafana
      dockerfile: Dockerfile
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      GF_SERVER_HTTP_ADDR: 0.0.0.0
      GF_SERVER_HTTP_PORT: 3000
    networks:
      - mynetwork
    volumes:
      - grafana_db:/var/lib/grafana
    restart: always

volumes:
  spark_checkpoints:
  spark_data:
  pg_data:
  airflow_logs:
  airflow_plugins:
  grafana_db: