daymet-national-parks/docker-compose.yml at main · d4mp3/daymet-national-parks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
services:

  postgres:
    build: './docker/postgres'
    restart: always
    container_name: postgres
    logging:
      driver: "json-file"
      options:
        max-file: "5"
        max-size: "10m"
    ports:
      - "32769:5432"
    volumes:
      - ./mnt/postgres:/var/lib/postgresql/data/pgdata
    env_file:
      - .env
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
      - PGDATA=/var/lib/postgresql/data/pgdata
    healthcheck:
      test: [ "CMD", "pg_isready", "-q", "-d", "airflow", "-U", "airflow" ]
      timeout: 45s
      interval: 10s
      retries: 10
    networks:
      - airflow-network

  minio:
    image: minio/minio:RELEASE.2024-06-13T22-53-53Z
    container_name: minio
    hostname: minio
    restart: always
    volumes:
      - ./mnt/minio:/data
    ports:
      - 9000:9000
      - 9001:9001
    env_file:
      - .env
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
    networks:
      - airflow-network

  spark-master:
    image: airflow/spark-master
    build: ./docker/spark/master
    container_name: spark-master
    ports:
      - "8082:8080"
      - "7077:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark
    healthcheck:
      test: [ "CMD", "nc", "-z", "spark-master", "8082" ]
      timeout: 45s
      interval: 10s
      retries: 10
    networks:
      - airflow-network

  spark-worker:
    image: airflow/spark-worker
    build: ./docker/spark/worker
    container_name: spark-worker
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
    healthcheck:
      test: [ "CMD", "nc", "-z", "spark-worker", "8081" ]
      timeout: 45s
      interval: 10s
      retries: 10
    networks:
      - airflow-network

  airflow:
    build: ./docker/airflow
    restart: always
    container_name: airflow
    depends_on:
      - postgres
    env_file:
      - .env
    volumes:
      - ./mnt/airflow/airflow.cfg:/opt/airflow/airflow.cfg
      - ./mnt/airflow/dags:/opt/airflow/dags
    ports:
      - 8080:8080
    healthcheck:
      test: [ "CMD", "nc", "-z", "airflow", "8080" ]
      timeout: 45s
      interval: 10s
      retries: 10
    networks:
      - airflow-network

  metabase:
    image: metabase/metabase:v0.50.4
    container_name: metabase
    volumes:
      - ./mnt/metabase:/metabase-data
    ports:
      - 3000:3000
    env_file:
      - .env
    environment:
      MB_DB_TYPE: postgres
      MB_DB_DBNAME: metabase
      MB_DB_PORT: 5432
      MB_DB_USER: ${MB_DB_USER}
      MB_DB_PASS: ${MB_DB_PASS}
      MB_DB_HOST: postgres
    healthcheck:
      test: curl --fail -I http://localhost:3000/api/health || exit 1
      interval: 15s
      timeout: 5s
      retries: 5
    networks:
      - airflow-network

  docker-proxy:
    image: alpine/socat
    command: "TCP4-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock"
    ports:
      - "2376:2375"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - airflow-network

networks:
  airflow-network:
    name: airflow-network