docker-compose.yml

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    container_name: zookeeper
    restart: unless-stopped
    hostname: zookeeper
    ports:
      - "2181:2181"
    healthcheck:
      test: nc -z localhost 2181 || exit -1
      interval: 10s
      timeout: 5s
      retries: 3
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      # cleanup
      ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT: 3
      ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: 24
      ZOOKEEPER_MAX_SESSION_TIMEOUT: 40000
    volumes:
      - zookeeper-data:/var/lib/zookeeper/data
      - zookeeper-logs:/var/lib/zookeeper/log
    networks:
      - kafka-network

  kafka1:
    image: confluentinc/cp-kafka:7.4.0
    container_name: kafka1
    restart: unless-stopped
    hostname: kafka1
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "29092:29092"
    healthcheck:
      test: ["CMD", "bash", "-c", "echo > /dev/tcp/localhost/29092"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      
      # stale sessions timeouts
      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 18000
      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 12000
      
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092,PLAINTEXT_HOST://kafka1:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_COMPRESSION_TYPE: lz4
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN'
      KAFKA_LOG4J_LOGGERS: 'kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN,kafka.producer.async.DefaultEventHandler=WARN'
      
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_MIN_INSYNC_REPLICAS: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
    volumes:
      - kafka1-data:/var/lib/kafka/data
    networks:
      - kafka-network

  kafka2:
    image: confluentinc/cp-kafka:7.4.0
    container_name: kafka2
    restart: unless-stopped
    hostname: kafka2
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9093:9093"
      - "29093:29093"
    healthcheck:
      test: ["CMD", "bash", "-c", "echo > /dev/tcp/localhost/29093"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 18000
      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 12000
      
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,PLAINTEXT_HOST://0.0.0.0:29093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9093,PLAINTEXT_HOST://kafka2:29093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT

      KAFKA_COMPRESSION_TYPE: lz4
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN'
      KAFKA_LOG4J_LOGGERS: 'kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN,kafka.producer.async.DefaultEventHandler=WARN'
      
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_MIN_INSYNC_REPLICAS: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
    volumes:
      - kafka2-data:/var/lib/kafka/data
    networks:
      - kafka-network

  kafka3:
    image: confluentinc/cp-kafka:7.4.0
    container_name: kafka3
    restart: unless-stopped
    hostname: kafka3
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9094:9094"
      - "29094:29094"
    healthcheck:
      test: ["CMD", "bash", "-c", "echo > /dev/tcp/localhost/29094"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 18000
      KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 12000
      
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094,PLAINTEXT_HOST://0.0.0.0:29094
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9094,PLAINTEXT_HOST://kafka3:29094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT

      KAFKA_COMPRESSION_TYPE: lz4
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN'
      KAFKA_LOG4J_LOGGERS: 'kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN,kafka.producer.async.DefaultEventHandler=WARN'
      
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_MIN_INSYNC_REPLICAS: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
    volumes:
      - kafka3-data:/var/lib/kafka/data
    networks:
      - kafka-network

  # start kafka and initialize topic
  kafka-setup:
    image: confluentinc/cp-kafka:7.4.0
    depends_on:
      kafka1:
        condition: service_healthy
      kafka2:
        condition: service_healthy  
      kafka3:
        condition: service_healthy
    volumes:
      - ./src/config:/scripts
    command: bash /scripts/create_topic.sh
    networks:
      - kafka-network

  # Clickhouse
  clickhouse1:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse1
    hostname: clickhouse1
    environment:
      CLICKHOUSE_USER: admin
      CLICKHOUSE_PASSWORD: password
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - ./clickhouse/clickhouse1/config:/etc/clickhouse-server/config.d
      - ./clickhouse/clickhouse1/users:/etc/clickhouse-server/users.d
      - ./clickhouse/clickhouse1/data:/var/lib/clickhouse
      - ./clickhouse/clickhouse1/logs:/var/log/clickhouse-server
    ports:
      - "8130:8123"
      - "9010:9000"
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    depends_on:
      - keeper1
      - keeper2
      - keeper3
    networks:
      - kafka-network
    restart: unless-stopped

  clickhouse2:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse2
    hostname: clickhouse2
    environment:
      CLICKHOUSE_USER: admin
      CLICKHOUSE_PASSWORD: password
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - ./clickhouse/clickhouse2/config:/etc/clickhouse-server/config.d
      - ./clickhouse/clickhouse2/users:/etc/clickhouse-server/users.d
      - ./clickhouse/clickhouse2/data:/var/lib/clickhouse
      - ./clickhouse/clickhouse2/logs:/var/log/clickhouse-server
    ports:
      - "8131:8123"
      - "9011:9000"
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    depends_on:
      - keeper1
      - keeper2
      - keeper3
    networks:
      - kafka-network
    restart: unless-stopped

  clickhouse-init:
    image: alpine:latest
    container_name: clickhouse-init
    depends_on:
      clickhouse1:
        condition: service_started
      clickhouse2:
        condition: service_started
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./clickhouse/create_tables.sql:/clickhouse/create_tables.sql
    entrypoint: |
      sh -c '
        apk add --no-cache docker-cli curl
        
        CONTAINERS="clickhouse1 clickhouse2"
        INIT_SQL_FILE="/clickhouse/create_tables.sql"
        CREATE_USER_SQL="CREATE USER IF NOT EXISTS test IDENTIFIED WITH plaintext_password BY '\''password'\'' HOST ANY;"

        GRANT_SELECT_SQL="GRANT SELECT(timestamp, alert_id, info, reason, ip) ON default.replicated_blacklist_events TO test;"
        GRANT_INSERT_SQL="GRANT INSERT(user_id, timestamp, ip, alert_id, dst_port, info, reason) ON default.replicated_blacklist_events TO test;"
        GRANT_SELECT_SQL_default_table="GRANT SELECT(timestamp, alert_id, info, reason, ip) ON default.blacklist_events TO test;"
        GRANT_INSERT_SQL_default_table="GRANT INSERT(user_id, timestamp, ip, alert_id, dst_port, info, reason) ON default.blacklist_events TO test;"
    
        for CONTAINER in $$CONTAINERS; do
          echo "Waiting for $$CONTAINER to be available..."
          
          # Wait for ClickHouse to be ready
          until docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "SELECT 1" >/dev/null 2>&1; do
            echo "Still waiting for $$CONTAINER..."
            sleep 5
          done
          
          echo "$$CONTAINER is ready!"
          
          # Create test user
          docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "$$CREATE_USER_SQL"
          echo "Created test user in $$CONTAINER"

          # Grant SELECT permissions
          docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "$$GRANT_SELECT_SQL"
          docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "$$GRANT_SELECT_SQL_default_table"
          echo "Granted SELECT permissions to test user in $$CONTAINER"
          
          # Grant INSERT permissions
          docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "$$GRANT_INSERT_SQL"
          docker exec "$$CONTAINER" clickhouse-client --user admin --password password -q "$$GRANT_INSERT_SQL_default_table"
          echo "Granted INSERT permissions to test user in $$CONTAINER"
          
          # Execute initialization SQL
          if [ -f "$$INIT_SQL_FILE" ]; then
            docker exec -i "$$CONTAINER" clickhouse-client --user admin --password password --multiquery < "$$INIT_SQL_FILE"
            echo "Initialization script executed for $$CONTAINER"
          else
            echo "No initialization script found at $$INIT_SQL_FILE"
          fi
        done
        
        echo "All ClickHouse containers initialized successfully!"
      '
    networks:
      - kafka-network

  keeper1:
    image: clickhouse/clickhouse-keeper:latest
    container_name: keeper1
    hostname: keeper1
    volumes:
      - ./clickhouse/keeper1/config.xml:/etc/clickhouse-keeper/keeper_config.xml
    networks:
      - kafka-network

  keeper2:
    image: clickhouse/clickhouse-keeper:latest
    container_name: keeper2
    hostname: keeper2
    volumes:
      - ./clickhouse/keeper2/config.xml:/etc/clickhouse-keeper/keeper_config.xml
    networks:
      - kafka-network

  keeper3:
    image: clickhouse/clickhouse-keeper:latest
    container_name: keeper3
    hostname: keeper3
    volumes:
      - ./clickhouse/keeper3/config.xml:/etc/clickhouse-keeper/keeper_config.xml
    networks:
      - kafka-network
  
  redis-1:
    image: redis:8.2-m01-alpine
    container_name: redis-1
    command:
      - redis-server
      - --port
      - '7001'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7001.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-1'
      - --cluster-announce-port
      - '7001'
      - --cluster-announce-bus-port
      - '17001'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7001'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-1:/data

  redis-2:
    image: redis:8.2-m01-alpine
    container_name: redis-2
    command:
      - redis-server
      - --port
      - '7002'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7002.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-2'
      - --cluster-announce-port
      - '7002'
      - --cluster-announce-bus-port
      - '17002'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7002'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-2:/data

  redis-3:
    image: redis:8.2-m01-alpine
    container_name: redis-3
    command:
      - redis-server
      - --port
      - '7003'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7003.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-3'
      - --cluster-announce-port
      - '7003'
      - --cluster-announce-bus-port
      - '17003'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7003'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-3:/data

  redis-4:
    image: redis:8.2-m01-alpine
    container_name: redis-4
    command:
      - redis-server
      - --port
      - '7004'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7004.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-4'
      - --cluster-announce-port
      - '7004'
      - --cluster-announce-bus-port
      - '17004'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7004'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-4:/data

  redis-5:
    image: redis:8.2-m01-alpine
    container_name: redis-5
    command:
      - redis-server
      - --port
      - '7005'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7005.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-5'
      - --cluster-announce-port
      - '7005'
      - --cluster-announce-bus-port
      - '17005'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7005'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-5:/data

  redis-6:
    image: redis:8.2-m01-alpine
    container_name: redis-6
    command:
      - redis-server
      - --port
      - '7006'
      - --cluster-enabled
      - 'yes'
      - --cluster-config-file
      - 'nodes-7006.conf'
      - --cluster-node-timeout
      - '5000'
      - --appendonly
      - 'yes'
      - --cluster-announce-ip
      - 'redis-6'
      - --cluster-announce-port
      - '7006'
      - --cluster-announce-bus-port
      - '17006'
    healthcheck:
      interval: 10s
      retries: 3
      test:
        - CMD
        - redis-cli
        - -p
        - '7006'
        - ping
      timeout: 5s
    networks:
      - kafka-network
    volumes:
      - redis-data-6:/data

  redis-cluster-creator:
    image: redis:8.2-m01-alpine
    container_name: redis-cluster-creator
    command: >
      sh -c '
        echo "Waiting for all Redis nodes to be ready...";
        for i in 1 2 3 4 5 6; do
          port=$$((7000 + i));
          until redis-cli -h redis-$$i -p $$port ping | grep -q PONG; do
            echo "Waiting for redis-$$i on port $$port...";
            sleep 2;
          done;
          echo "redis-$$i is ready";
        done;

        echo "Checking if cluster already exists...";
        if redis-cli -h redis-1 -p 7001 cluster info | grep -q "cluster_state:ok"; then
          echo "Redis cluster already exists and is healthy";
          redis-cli -h redis-1 -p 7001 cluster info;
          redis-cli -h redis-1 -p 7001 cluster nodes;
          echo "Cluster setup completed (already existed)";
          exit 0;
        fi;

        echo "All Redis nodes are ready. Creating cluster...";
        redis-cli --cluster create \
          redis-1:7001 redis-2:7002 redis-3:7003 redis-4:7004 redis-5:7005 redis-6:7006 \
          --cluster-replicas 1 --cluster-yes;
        if [ $$? -eq 0 ]; then
          echo "Redis cluster created successfully";
          exit 0;
        else
          echo "Failed to create Redis cluster";
          exit 1;
        fi;
        echo "Verifying cluster status...";
        redis-cli -h redis-1 -p 7001 cluster info;
        redis-cli -h redis-1 -p 7001 cluster nodes;
        echo "Cluster setup completed";
        exit 0;
      '
    depends_on:
      redis-1:
        condition: service_healthy
      redis-2:
        condition: service_healthy
      redis-3:
        condition: service_healthy
      redis-4:
        condition: service_healthy
      redis-5:
        condition: service_healthy
      redis-6:
        condition: service_healthy
    networks:
      - kafka-network
    restart: "no"

  api_server:
    build:
      context: ./src
      dockerfile: api_server/Dockerfile
    container_name: api_server
    depends_on:
      - kafka1
      - clickhouse-init
      - redis-cluster-creator
    ports:
      - "8000:8000"
    networks:
      - kafka-network
    volumes:
      - ./src:/app
    command: uvicorn api_server.main:app --host 0.0.0.0 --port 8000

  producer:
    build:
      context: ./src
      dockerfile: producer/Dockerfile
    depends_on:
      clickhouse-init:
        condition: service_completed_successfully
      kafka-setup:
        condition: service_completed_successfully
    networks:
      - kafka-network
    volumes:
      - ./src:/app
    command: python producer/producer.py

  consumer:
    build:
      context: ./src
      dockerfile: consumer/Dockerfile
    restart: unless-stopped
    depends_on:
      kafka1:
        condition: service_healthy
      kafka2:
        condition: service_healthy
      kafka3:
        condition: service_healthy
      redis-cluster-creator:
        condition: service_completed_successfully
      clickhouse-init:
        condition: service_completed_successfully
      kafka-setup:
        condition: service_completed_successfully
    networks:
      - kafka-network
    volumes:
      - ./src:/app
    command: python -u consumer/consumer.py


networks:
  kafka-network:
    driver: bridge

volumes:
  zookeeper-data:
  zookeeper-logs:
  kafka1-data:
  kafka2-data:
  kafka3-data:
  clickhouse1-data:
  clickhouse1-logs:
  clickhouse2-data:
  clickhouse2-logs:
  redis-data-1:
  redis-data-2:
  redis-data-3:
  redis-data-4:
  redis-data-5:
  redis-data-6: