Skip to content

Commit 92535d2

Browse files
skudasovskudasov
committed
add Prometheus target and verify plugin metrics
1 parent e515a9e commit 92535d2

File tree

6 files changed

+179
-7
lines changed

6 files changed

+179
-7
lines changed

‎framework/.changeset/v0.6.3.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
- Add `otel-collector` and connect it with `Loki` and `Tempo`
1+
- Add otel-collector and connect it with Loki, Tempo and Prometheus
2+
- Tune Tempo, verify we can store traces for hours
3+
- Add basic tests to debug metrics, logs and traces endpoints

‎framework/cmd/observability/compose/conf/prometheus.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ global:
22
scrape_interval: 10s
33

44
scrape_configs:
5+
- job_name: 'otel-collector'
6+
scrape_interval: 10s
7+
static_configs:
8+
- targets: [ 'otel-collector:8889' ]
59
- job_name: 'ctf'
610
metrics_path: /metrics
711
docker_sd_configs:

‎framework/cmd/observability/compose/docker-compose.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ services:
2020
command:
2121
- '--config=/etc/otel/config.yaml'
2222
ports:
23-
- "4317:4317" #grpc
24-
- "4318:4318" #http
23+
- "4317:4317" # grpc
24+
- "4318:4318" # http
25+
- "8889:8889" # Prometheus scrape target
2526
depends_on:
2627
- loki
2728
cadvisor:

‎framework/cmd/observability/compose/otel.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,20 @@ exporters:
2222
endpoint: "http://tempo:4317"
2323
tls:
2424
insecure: true
25+
prometheus:
26+
endpoint: "0.0.0.0:8889"
2527

2628
service:
2729
pipelines:
2830
traces:
2931
receivers: [otlp]
32+
processors: [batch]
3033
exporters: [debug, otlp]
3134
logs:
3235
receivers: [otlp]
3336
processors: [batch]
34-
exporters: [debug, otlphttp/logs]
37+
exporters: [debug, otlphttp/logs]
38+
metrics:
39+
receivers: [otlp]
40+
processors: [batch]
41+
exporters: [debug, prometheus]

‎framework/cmd/observability/compose/tempo.yaml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,18 @@ distributor:
88
grpc:
99
http:
1010

11+
overrides:
12+
max_traces_per_user: 50000
13+
1114
storage:
1215
trace:
1316
backend: local
1417
local:
1518
path: /tmp/tempo/blocks
1619
wal:
17-
path: /tmp/tempo/wal
20+
path: /tmp/tempo/wal
21+
22+
compactor:
23+
compaction:
24+
max_block_bytes: 1073741824 # 1GB max block size
25+
max_compaction_objects: 1000000

‎framework/cmd/otel_test.go‎

Lines changed: 152 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,31 @@ import (
1818
"go.opentelemetry.io/otel/trace"
1919
)
2020

21-
func TestSendLogsToOTELCollector(t *testing.T) {
21+
func TestOTELSendMetricsToOTELCollector(t *testing.T) {
22+
t.Skip("run manually to debug otel-collector")
23+
res, err := resource.New(context.Background(),
24+
resource.WithAttributes(
25+
semconv.ServiceName("test-metrics-service"),
26+
semconv.ServiceVersion("1.0.0"),
27+
attribute.String("environment", "test"),
28+
),
29+
)
30+
if err != nil {
31+
t.Fatalf("failed to create resource: %v", err)
32+
}
33+
34+
metrics := generateMetricsData(res)
35+
jsonData, _ := json.MarshalIndent(metrics, "", " ")
36+
t.Logf("Sending metrics payload:\n%s\n", string(jsonData))
37+
38+
err = sendMetricsToCollector(metrics)
39+
if err != nil {
40+
t.Fatalf("failed to send metrics: %v", err)
41+
}
42+
t.Log("Metrics sent successfully to OTEL Collector!")
43+
}
44+
45+
func TestOTELSendLogsToOTELCollector(t *testing.T) {
2246
t.Skip("run manually to debug otel-collector")
2347
res, err := resource.New(context.Background(),
2448
resource.WithAttributes(
@@ -39,7 +63,7 @@ func TestSendLogsToOTELCollector(t *testing.T) {
3963
t.Log("Logs sent successfully!")
4064
}
4165

42-
func TestSendTracesToOTELCollector(t *testing.T) {
66+
func TestOTELSendTracesToOTELCollector(t *testing.T) {
4367
t.Skip("run manually to debug otel-collector")
4468
res, err := resource.New(context.Background(),
4569
resource.WithAttributes(
@@ -61,6 +85,104 @@ func TestSendTracesToOTELCollector(t *testing.T) {
6185
t.Log("Traces sent successfully to OTEL Collector!")
6286
}
6387

88+
func generateMetricsData(res *resource.Resource) map[string]interface{} {
89+
now := time.Now()
90+
nanos := now.UnixNano()
91+
startTime := now.Add(-5 * time.Minute).UnixNano()
92+
93+
return map[string]interface{}{
94+
"resourceMetrics": []interface{}{
95+
map[string]interface{}{
96+
"resource": map[string]interface{}{
97+
"attributes": resourceToAttributes(res),
98+
},
99+
"scopeMetrics": []interface{}{
100+
map[string]interface{}{
101+
"scope": map[string]interface{}{
102+
"name": "test-metrics",
103+
"version": "1.0",
104+
},
105+
"metrics": []interface{}{
106+
// Counter example
107+
map[string]interface{}{
108+
"name": "http.requests",
109+
"description": "Count of HTTP requests",
110+
"unit": "1",
111+
"sum": map[string]interface{}{
112+
"dataPoints": []interface{}{
113+
map[string]interface{}{
114+
"startTimeUnixNano": startTime,
115+
"timeUnixNano": nanos,
116+
"asInt": "42",
117+
"attributes": []interface{}{
118+
map[string]interface{}{
119+
"key": "http.method",
120+
"value": map[string]interface{}{"stringValue": "GET"},
121+
},
122+
map[string]interface{}{
123+
"key": "http.status",
124+
"value": map[string]interface{}{"stringValue": "200"},
125+
},
126+
},
127+
},
128+
},
129+
"aggregationTemporality": 2, // Cumulative
130+
"isMonotonic": true,
131+
},
132+
},
133+
// Gauge example
134+
map[string]interface{}{
135+
"name": "memory.usage",
136+
"description": "Memory usage in bytes",
137+
"unit": "By",
138+
"gauge": map[string]interface{}{
139+
"dataPoints": []interface{}{
140+
map[string]interface{}{
141+
"timeUnixNano": nanos,
142+
"asDouble": 1024.42,
143+
"attributes": []interface{}{
144+
map[string]interface{}{
145+
"key": "memory.type",
146+
"value": map[string]interface{}{"stringValue": "heap"},
147+
},
148+
},
149+
},
150+
},
151+
},
152+
},
153+
// Histogram example
154+
map[string]interface{}{
155+
"name": "http.request.duration",
156+
"description": "HTTP request duration in seconds",
157+
"unit": "s",
158+
"histogram": map[string]interface{}{
159+
"dataPoints": []interface{}{
160+
map[string]interface{}{
161+
"startTimeUnixNano": startTime,
162+
"timeUnixNano": nanos,
163+
"count": "5",
164+
"sum": 4.2,
165+
"bucketCounts": []interface{}{"0", "1", "3", "1", "0"},
166+
"explicitBounds": []interface{}{"0.1", "0.5", "1", "2"},
167+
"attributes": []interface{}{
168+
map[string]interface{}{
169+
"key": "http.route",
170+
"value": map[string]interface{}{"stringValue": "/api/users"},
171+
},
172+
},
173+
},
174+
},
175+
"aggregationTemporality": 2, // Cumulative
176+
},
177+
},
178+
},
179+
},
180+
},
181+
},
182+
},
183+
}
184+
}
185+
64186
func generateLogRecords(res *resource.Resource) map[string]interface{} {
65187
now := time.Now()
66188
nanos := now.UnixNano()
@@ -205,6 +327,34 @@ func resourceToAttributes(res *resource.Resource) []interface{} {
205327
return attrs
206328
}
207329

330+
func sendMetricsToCollector(metrics map[string]interface{}) error {
331+
jsonData, err := json.Marshal(metrics)
332+
if err != nil {
333+
return fmt.Errorf("failed to marshal metrics: %w", err)
334+
}
335+
336+
req, err := http.NewRequest("POST", "http://localhost:4318/v1/metrics", bytes.NewBuffer(jsonData))
337+
if err != nil {
338+
return fmt.Errorf("failed to create request: %w", err)
339+
}
340+
341+
req.Header.Set("Content-Type", "application/json")
342+
343+
client := &http.Client{}
344+
resp, err := client.Do(req)
345+
if err != nil {
346+
return fmt.Errorf("failed to send request: %w", err)
347+
}
348+
defer resp.Body.Close()
349+
350+
if resp.StatusCode >= 300 {
351+
body, _ := io.ReadAll(resp.Body)
352+
return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
353+
}
354+
355+
return nil
356+
}
357+
208358
func sendLogsToCollector(logs map[string]interface{}) error {
209359
jsonData, err := json.Marshal(logs)
210360
if err != nil {

0 commit comments

Comments
 (0)