ALICE_Data_Lakehouse/docker-compose.yaml at main · Przemyslaw11/ALICE_Data_Lakehouse · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
services:
 postgres:
   image: postgres:13
   platform: linux/amd64
   hostname: postgres
   container_name: postgres
   restart: always
   volumes:
     - ./data/pgdata:/var/lib/postgresql/data
   environment:
     - POSTGRES_DB=metastore
     - POSTGRES_USER=postgres
     - POSTGRES_PASSWORD=postgres
   healthcheck:
     test: ["CMD-SHELL", "pg_isready -U postgres -d metastore"]
     interval: 5s
     retries: 10
   ports:
     - "5432:5432"
   networks:
     - dldg

 metastore:
   build:
     context: .
     dockerfile: Dockerfile
   platform: linux/amd64
   hostname: metastore
   image: my-hive-metastore:latest
   container_name: metastore
   restart: always
   environment:
     - SERVICE_NAME=metastore
     - DB_DRIVER=postgres
     - METASTORE_DB_HOSTNAME=postgres
     - METASTORE_DB_PORT=5432
     - METASTORE_DB_NAME=metastore
     - METASTORE_DB_USER=postgres
     - METASTORE_DB_PASSWORD=postgres
   ports:
     - "9083:9083"
   depends_on:
     postgres:
       condition: service_healthy
   networks:
     - dldg

 trinodb:
   restart: always
   image: trinodb/trino:426
   platform: linux/amd64
   hostname: trinodb
   container_name: trinodb
   volumes:
     - ./etc/catalog/delta.properties:/etc/trino/catalog/delta.properties
     - ./conf:/etc/hadoop/conf/
   ports:
     - "9090:8080"
   environment:
     - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY}
     - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY}
     - AWS_ENDPOINT=https://s3p.cloud.cyfronet.pl
     - AWS_DEFAULT_REGION=us-west-2
     - TRINO_ENDPOINT=http://trinodb:8080
   depends_on:
     - metastore
   networks:
     - dldg

 spark:
   image: bitnami/spark:3.4.1
   platform: linux/amd64
   hostname: spark
   container_name: spark
   environment:
     - SPARK_MODE=master
     - SPARK_MASTER_HOST=spark
     - SPARK_MASTER_PORT=7077
     - SPARK_MASTER_WEBUI_PORT=8080
   ports:
     - "8080:8080"
     - "7077:7077"
   volumes:
     - ./conf:/opt/spark/conf
     - ./jars:/opt/spark/jars
   networks:
     - dldg
   depends_on:
     - metastore

 spark-worker:
   image: bitnami/spark:3.4.1
   platform: linux/amd64
   environment:
     - SPARK_MODE=worker
     - SPARK_MASTER_HOST=spark
     - SPARK_MASTER_PORT=7077
     - SPARK_WORKER_MEMORY=1G
     - SPARK_WORKER_CORES=1
   volumes:
     - ./conf:/opt/spark/conf
     - ./jars:/opt/spark/jars
   networks:
     - dldg
   depends_on:
     - spark

networks:
 dldg:
   name: dldg