Skip to content

Commit d4ac66d

Browse files
jpraynaudjpraynaud
authored
Merge pull request #1128 from input-output-hk/jpraynaud/1122-ingest-metrics-logs-in-grafana
Ingest metrics/logs in Grafana
2 parents 1853b2f + 91c4d3e commit d4ac66d

File tree

8 files changed

+108
-11
lines changed

8 files changed

+108
-11
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,8 +453,14 @@ jobs:
453453
ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
454454
PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
455455
PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
456+
PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
457+
PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
458+
PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
456459
LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
457460
LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
461+
LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
462+
LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
463+
LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
458464

459465
defaults:
460466
run:
@@ -496,8 +502,14 @@ jobs:
496502
mithril_signers = ${{ matrix.mithril_signers }}
497503
prometheus_auth_username = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
498504
prometheus_auth_password = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
505+
prometheus_ingest_host = "${{ env.PROMETHEUS_INGEST_HOST }}"
506+
prometheus_ingest_username = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
507+
prometheus_ingest_password = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
499508
loki_auth_username = "${{ env.LOKI_AUTH_USERNAME }}"
500509
loki_auth_password = "${{ env.LOKI_AUTH_PASSWORD }}"
510+
loki_ingest_host = "${{ env.LOKI_INGEST_HOST }}"
511+
loki_ingest_username = "${{ env.LOKI_INGEST_USERNAME }}"
512+
loki_ingest_password = "${{ env.LOKI_INGEST_PASSWORD }}"
501513
EOF
502514
terraform fmt ./env.variables.tfvars
503515
cat ./env.variables.tfvars

‎.github/workflows/pre-release.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,14 @@ jobs:
181181
ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
182182
PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
183183
PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
184+
PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
185+
PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
186+
PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
184187
LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
185188
LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
189+
LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
190+
LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
191+
LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
186192

187193
defaults:
188194
run:
@@ -224,8 +230,14 @@ jobs:
224230
mithril_signers = ${{ matrix.mithril_signers }}
225231
prometheus_auth_username = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
226232
prometheus_auth_password = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
233+
prometheus_ingest_host = "${{ env.PROMETHEUS_INGEST_HOST }}"
234+
prometheus_ingest_username = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
235+
prometheus_ingest_password = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
227236
loki_auth_username = "${{ env.LOKI_AUTH_USERNAME }}"
228237
loki_auth_password = "${{ env.LOKI_AUTH_PASSWORD }}"
238+
loki_ingest_host = "${{ env.LOKI_INGEST_HOST }}"
239+
loki_ingest_username = "${{ env.LOKI_INGEST_USERNAME }}"
240+
loki_ingest_password = "${{ env.LOKI_INGEST_PASSWORD }}"
229241
EOF
230242
terraform fmt ./env.variables.tfvars
231243
cat ./env.variables.tfvars

‎.github/workflows/release.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,8 +173,14 @@ jobs:
173173
ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
174174
PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
175175
PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
176+
PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
177+
PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
178+
PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
176179
LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
177180
LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
181+
LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
182+
LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
183+
LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
178184

179185
defaults:
180186
run:
@@ -217,8 +223,14 @@ jobs:
217223
mithril_signers = ${{ matrix.mithril_signers }}
218224
prometheus_auth_username = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
219225
prometheus_auth_password = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
226+
prometheus_ingest_host = "${{ env.PROMETHEUS_INGEST_HOST }}"
227+
prometheus_ingest_username = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
228+
prometheus_ingest_password = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
220229
loki_auth_username = "${{ env.LOKI_AUTH_USERNAME }}"
221230
loki_auth_password = "${{ env.LOKI_AUTH_PASSWORD }}"
231+
loki_ingest_host = "${{ env.LOKI_INGEST_HOST }}"
232+
loki_ingest_username = "${{ env.LOKI_INGEST_USERNAME }}"
233+
loki_ingest_password = "${{ env.LOKI_INGEST_PASSWORD }}"
222234
EOF
223235
terraform fmt ./env.variables.tfvars
224236
cat ./env.variables.tfvars

‎mithril-infra/assets/docker/prometheus/prometheus.yml‎ renamed to ‎mithril-infra/assets/docker/prometheus/prometheus-base.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,3 @@ scrape_configs:
2727
file_sd_configs:
2828
- files:
2929
- 'cardano.json'
30-
31-

‎mithril-infra/assets/docker/promtail/promtail-config.yml‎ renamed to ‎mithril-infra/assets/docker/promtail/promtail-config-base.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@ server:
55
positions:
66
filename: /tmp/positions.yaml
77

8-
clients:
9-
- url: http://loki:3100/loki/api/v1/push
10-
118
scrape_configs:
129
- job_name: system
1310
static_configs:
@@ -49,4 +46,7 @@ scrape_configs:
4946
image_id:
5047
container_id:
5148
- output:
52-
source: output
49+
source: output
50+
51+
clients:
52+
- url: http://loki:3100/loki/api/v1/push

‎mithril-infra/assets/infra.version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.1.9
1+
0.1.10

‎mithril-infra/mithril.monitoring.tf‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,20 @@ resource "null_resource" "mithril_monitoring" {
2828
"mkdir -p /home/curry/data/monitoring/prometheus",
2929
"mkdir -p /home/curry/data/monitoring/loki",
3030
<<-EOT
31+
set -e
32+
# Copy prometheus base configuration
33+
cp /home/curry/docker/prometheus/prometheus-base.yml /home/curry/docker/prometheus/prometheus.yml
34+
# Setup prometheus remote write
35+
if [ -n "${var.prometheus_ingest_host}" ] ; then
36+
cat >> /home/curry/docker/prometheus/prometheus.yml << EOF
37+
38+
remote_write:
39+
- url: https://${var.prometheus_ingest_host}/api/prom/push
40+
basic_auth:
41+
username: ${var.prometheus_ingest_username}
42+
password: ${var.prometheus_ingest_password}
43+
EOF
44+
fi
3145
# Setup prometheus targets configuration for Cardano nodes
3246
CARDANO_NODES=$(docker ps --format='{{.Names}}:12798,' | grep "cardano-node" | sort | tr -d '\n\t\r ' | sed 's/.$//')
3347
cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDANO_NODES" '. += [{
@@ -38,6 +52,19 @@ cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDAN
3852
}]' | jq '. | map(try(.targets |= split(",")) // .)' > /home/curry/docker/prometheus/cardano.json.new
3953
rm -f /home/curry/docker/prometheus/cardano.json
4054
mv /home/curry/docker/prometheus/cardano.json.new docker/prometheus/cardano.json
55+
EOT
56+
,
57+
<<-EOT
58+
set -e
59+
# Copy promtail base configuration
60+
cp /home/curry/docker/promtail/promtail-config-base.yml /home/curry/docker/promtail/promtail-config.yml
61+
# Setup promtail remote client
62+
if [ -n "${var.loki_ingest_host}" ] ; then
63+
cat >> /home/curry/docker/promtail/promtail-config.yml << EOF
64+
65+
- url: https://${var.loki_ingest_username}:${var.loki_ingest_password}@${var.loki_ingest_host}/loki/api/v1/push
66+
EOF
67+
fi
4168
EOT
4269
]
4370
}

‎mithril-infra/variables.tf‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,25 +174,61 @@ variable "mithril_aggregator_auth_password" {
174174

175175
variable "prometheus_auth_username" {
176176
type = string
177-
description = "The username for authentication on prometheus"
177+
description = "The username for authentication on local prometheus endpoint"
178178
default = ""
179179
}
180180

181181
variable "prometheus_auth_password" {
182182
type = string
183-
description = "The password for authentication on prometheus"
183+
description = "The password for authentication on local prometheus endpoint"
184+
default = ""
185+
}
186+
187+
variable "prometheus_ingest_host" {
188+
type = string
189+
description = "The host to ingest on remote prometheus endpoint"
190+
default = ""
191+
}
192+
193+
variable "prometheus_ingest_username" {
194+
type = string
195+
description = "The username to ingest on remote prometheus endpoint"
196+
default = ""
197+
}
198+
199+
variable "prometheus_ingest_password" {
200+
type = string
201+
description = "The password to ingest on remote prometheus endpoint"
184202
default = ""
185203
}
186204

187205
variable "loki_auth_username" {
188206
type = string
189-
description = "The username for authentication on loki"
207+
description = "The username for authentication on local loki endpoint"
190208
default = ""
191209
}
192210

193211
variable "loki_auth_password" {
194212
type = string
195-
description = "The password for authentication on loki"
213+
description = "The password for authentication on local loki endpoint"
214+
default = ""
215+
}
216+
217+
variable "loki_ingest_host" {
218+
type = string
219+
description = "The host to ingest on remote loki endpoint"
220+
default = ""
221+
}
222+
223+
variable "loki_ingest_username" {
224+
type = string
225+
description = "The username to ingest on remote loki endpoint"
226+
default = ""
227+
}
228+
229+
variable "loki_ingest_password" {
230+
type = string
231+
description = "The password to ingest on remote loki endpoint"
196232
default = ""
197233
}
198234

0 commit comments

Comments
 (0)