Skip to content

Commit cc42a24

Browse files
michaelloosenmichaelloosen
authored
Merge pull request #213 from jembi/logging
Logging storage service for platform monitoring
2 parents d0e0e7f + 42226fb commit cc42a24

File tree

21 files changed

+1363
-60
lines changed

21 files changed

+1363
-60
lines changed

‎.env.cluster‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
2828
# Reverse Proxy - Nginx
2929
REVERSE_PROXY_INSTANCES=3
3030
DOMAIN_NAME=domain
31-
SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain
31+
SUBDOMAINS=openhimcomms.domain,openhimcore.domain,openhimconsole.domain,kibana.domain,reports.domain,santewww.domain,santempi.domain,superset.domain,keycloak.domain,grafana.domain,minio.domain
3232
STAGING=true
3333
INSECURE=false
3434

‎.github/workflows/run-tests.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ else
4949
DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":hapi
5050
elif [[ $folder_name == *"santempi"* ]]; then
5151
DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":sante
52+
elif [[ $folder_name == *"monitoring"* ]]; then
53+
DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":monitoring
5254
fi
5355
done
5456
fi

‎identity-access-manager-keycloak/docker-compose.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,14 @@ version: '3.9'
22

33
services:
44
identity-access-manager-keycloak:
5-
image: keycloak/keycloak
6-
command: ["start", "--proxy=edge", "--hostname-url=${KC_FRONTEND_URL}", "--import-realm"]
5+
image: keycloak/keycloak:20.0
6+
command:
7+
[
8+
"start",
9+
"--proxy=edge",
10+
"--hostname-url=${KC_FRONTEND_URL}",
11+
"--import-realm"
12+
]
713
hostname: identity-access-manager-keycloak
814
healthcheck:
915
test: curl --fail http://localhost:8080/health/ready || exit 1

‎infrastructure/ansible/roles/docker/files/docker-daemon.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"log-driver": "json-file",
33
"log-opts": {
44
"max-size": "10m",
5-
"max-file": "3"
5+
"max-file": "3",
6+
"labels-regex": "^.+"
67
}
78
}

‎monitoring/docker-compose.cluster.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,96 @@ services:
2929
- '--web.console.templates=/etc/prometheus/consoles'
3030
- '--web.enable-lifecycle'
3131

32+
minio-01:
33+
deploy:
34+
placement:
35+
constraints:
36+
- "node.labels.name==node-1"
37+
38+
minio-02:
39+
image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
40+
entrypoint: sh
41+
command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
42+
environment:
43+
MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
44+
MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
45+
healthcheck:
46+
test:
47+
[
48+
"CMD",
49+
"curl",
50+
"-f",
51+
"http://localhost:9000/minio/health/live"
52+
]
53+
interval: 30s
54+
timeout: 20s
55+
retries: 3
56+
hostname: minio-02
57+
volumes:
58+
- minio-02-data1:/data1
59+
- minio-02-data2:/data2
60+
deploy:
61+
placement:
62+
constraints:
63+
- "node.labels.name==node-1"
64+
replicas: 1
65+
66+
minio-03:
67+
image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
68+
entrypoint: sh
69+
command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
70+
environment:
71+
MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
72+
MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
73+
healthcheck:
74+
test:
75+
[
76+
"CMD",
77+
"curl",
78+
"-f",
79+
"http://localhost:9000/minio/health/live"
80+
]
81+
interval: 30s
82+
timeout: 20s
83+
retries: 3
84+
hostname: minio-03
85+
volumes:
86+
- minio-03-data1:/data1
87+
- minio-03-data2:/data2
88+
deploy:
89+
placement:
90+
constraints:
91+
- "node.labels.name==node-2"
92+
replicas: 1
93+
94+
minio-04:
95+
image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
96+
entrypoint: sh
97+
command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
98+
environment:
99+
MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
100+
MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
101+
healthcheck:
102+
test:
103+
[
104+
"CMD",
105+
"curl",
106+
"-f",
107+
"http://localhost:9000/minio/health/live"
108+
]
109+
interval: 30s
110+
timeout: 20s
111+
retries: 3
112+
hostname: minio-04
113+
volumes:
114+
- minio-04-data1:/data1
115+
- minio-04-data2:/data2
116+
deploy:
117+
placement:
118+
constraints:
119+
- "node.labels.name==node-3"
120+
replicas: 1
121+
32122
configs:
33123
prometheus.yml:
34124
file: ./prometheus/prometheus.yml
@@ -38,3 +128,9 @@ configs:
38128

39129
volumes:
40130
prometheus_data_backup:
131+
minio-02-data1:
132+
minio-02-data2:
133+
minio-03-data1:
134+
minio-03-data2:
135+
minio-04-data1:
136+
minio-04-data2:

‎monitoring/docker-compose.dev.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,15 @@ services:
1212
- target: 9090
1313
published: 9090
1414
mode: host
15+
16+
loki:
17+
ports:
18+
- target: 3100
19+
published: 3100
20+
mode: host
21+
22+
minio-01:
23+
ports:
24+
- target: 9001
25+
published: 9005
26+
mode: host

‎monitoring/docker-compose.yml‎

Lines changed: 75 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ services:
44
grafana:
55
image: grafana/grafana-oss:9.2.3
66
volumes:
7-
- grafana_data:/var/lib/grafana
7+
- grafana-data:/var/lib/grafana
88
environment:
99
GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER}
1010
GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD}
@@ -49,12 +49,14 @@ services:
4949
source: kminion-groups_rev1.json
5050
- target: /etc/grafana/provisioning/dashboards/applications/kminion-topic_rev1.json
5151
source: kminion-topic_rev1.json
52+
- target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json
53+
source: logging-universal-dashboard_rev1.json
5254

5355
prometheus:
5456
image: prom/prometheus:v2.38.0
5557
user: root
5658
volumes:
57-
- prometheus_data:/prometheus
59+
- prometheus-data:/prometheus
5860
- /var/run/docker.sock:/var/run/docker.sock:ro
5961
configs:
6062
- target: /etc/prometheus/prometheus.yml
@@ -88,11 +90,58 @@ services:
8890
deploy:
8991
mode: global
9092

91-
prometheus-kafka-adapter:
92-
image: telefonica/prometheus-kafka-adapter:1.8.0
93+
loki:
94+
image: grafana/loki:2.6.1
95+
volumes:
96+
- loki-data:/tmp/loki
97+
environment:
98+
MO_SECURITY_ADMIN_USER: ${MO_SECURITY_ADMIN_USER}
99+
MO_SECURITY_ADMIN_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
100+
configs:
101+
- target: /etc/loki/loki-config.yml
102+
source: loki-config.yml
103+
command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
104+
deploy:
105+
labels:
106+
- prometheus-job-service=loki
107+
- prometheus-address=loki:3100
108+
109+
promtail:
110+
image: grafana/promtail:2.6.1
111+
volumes:
112+
- /var/lib/docker/containers:/host/containers
113+
- /var/log:/var/log:ro
114+
configs:
115+
- target: /etc/promtail/promtail-config.yml
116+
source: promtail-config.yml
117+
command: -config.file=/etc/promtail/promtail-config.yml
118+
deploy:
119+
mode: global
120+
121+
minio-01:
122+
image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
123+
entrypoint: sh
124+
command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}'
93125
environment:
94-
- KAFKA_BROKER_LIST=kafka:9092
95-
- KAFKA_COMPRESSION=gzip
126+
MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
127+
MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
128+
healthcheck:
129+
test:
130+
[
131+
"CMD",
132+
"curl",
133+
"-f",
134+
"http://localhost:9000/minio/health/live"
135+
]
136+
interval: 30s
137+
timeout: 20s
138+
retries: 3
139+
hostname: minio-01
140+
volumes:
141+
- minio-01-data1:/data1
142+
- minio-01-data2:/data2
143+
deploy:
144+
replicas: 1
96145

97146
configs:
98147
grafana.ini:
@@ -135,12 +184,30 @@ configs:
135184
name: kminion-topic_rev1.json-${kminion_topic_rev1_json_DIGEST:?err}
136185
labels:
137186
name: grafana
187+
logging-universal-dashboard_rev1.json:
188+
file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json
189+
name: logging-universal-dashboard_rev1.json-${logging_universal_dashboard_rev1_json_DIGEST:?err}
190+
labels:
191+
name: grafana
138192
prometheus.yml:
139193
file: ./prometheus/prometheus.yml
140194
name: prometheus.yml-${prometheus_yml_DIGEST:?err}
141195
labels:
142196
name: prometheus
197+
loki-config.yml:
198+
file: ./loki/loki-config.yml
199+
name: loki-config.yml-${loki_config_yml_DIGEST:?err}
200+
labels:
201+
name: loki
202+
promtail-config.yml:
203+
file: ./promtail/promtail-config.yml
204+
name: promtail-config.yml-${promtail_config_yml_DIGEST:?err}
205+
labels:
206+
name: promtail
143207

144208
volumes:
145-
prometheus_data:
146-
grafana_data:
209+
prometheus-data:
210+
grafana-data:
211+
loki-data:
212+
minio-01-data1:
213+
minio-01-data2:

0 commit comments

Comments
 (0)