Skip to content

Commit 65e9307

Browse files
takeokunntakeokunn
committed
feat(formats): add OpenTelemetry Collector log format and converter
- Add otel_collector_log.json format definition for OpenTelemetry Collector - Add otel_collector_log-converter.sh script for flattening OTEL logs - Register new format and script in build system - Add test log file and update test Makefile for OTEL Collector logs - Improve error handling and argument validation in pcap_log-converter.sh
1 parent 1445f87 commit 65e9307

File tree

6 files changed

+169
-0
lines changed

6 files changed

+169
-0
lines changed

‎src/formats/formats.am‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ FORMAT_FILES = \
3636
$(srcdir)/%reldir%/openam_log.json \
3737
$(srcdir)/%reldir%/openamdb_log.json \
3838
$(srcdir)/%reldir%/openstack_log.json \
39+
$(srcdir)/%reldir%/otel_collector_log.json \
3940
$(srcdir)/%reldir%/otlp_python_log.json \
4041
$(srcdir)/%reldir%/page_log.json \
4142
$(srcdir)/%reldir%/pcap_log.json \

‎src/formats/otel_collector_log.json‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
3+
"otel_collector_log": {
4+
"title": "OpenTelemetry Collector File Exporter",
5+
"description": "Format for OpenTelemetry Collector file exporter JSON logs",
6+
"url": "https://opentelemetry.io/docs/specs/otel/protocol/file-exporter/",
7+
"file-type": "json",
8+
"convert-to-local-time": true,
9+
"converter": {
10+
"header": {
11+
"expr": {
12+
"otel_collector": ":header REGEXP '.*\"resourceLogs\".*'"
13+
},
14+
"size": 100
15+
},
16+
"command": "otel_collector_log-converter.sh"
17+
},
18+
"line-format": [
19+
{
20+
"field": "__timestamp__"
21+
},
22+
" ",
23+
{
24+
"field": "__level__",
25+
"text-transform": "uppercase",
26+
"min-width": 5
27+
},
28+
" ",
29+
{
30+
"field": "service.name",
31+
"default-value": "-",
32+
"auto-width": true
33+
},
34+
" ",
35+
{
36+
"field": "body"
37+
},
38+
{
39+
"field": "trace_id",
40+
"prefix": " [trace:",
41+
"suffix": "]",
42+
"default-value": ""
43+
}
44+
],
45+
"level-field": "severity",
46+
"level": {
47+
"fatal": "FATAL",
48+
"error": "ERROR",
49+
"warning": "WARN",
50+
"info": "INFO",
51+
"debug": "DEBUG",
52+
"trace": "TRACE"
53+
},
54+
"timestamp-field": "timestamp_ns",
55+
"timestamp-divisor": 1000000,
56+
"body-field": "body",
57+
"opid-field": "trace_id",
58+
"hide-extra": true,
59+
"value": {
60+
"timestamp_ns": {
61+
"kind": "string",
62+
"hidden": true
63+
},
64+
"observed_ns": {
65+
"kind": "string",
66+
"hidden": true
67+
},
68+
"severity_number": {
69+
"kind": "integer",
70+
"identifier": true
71+
},
72+
"severity": {
73+
"kind": "string",
74+
"identifier": true
75+
},
76+
"body": {
77+
"kind": "string"
78+
},
79+
"trace_id": {
80+
"kind": "string",
81+
"identifier": true
82+
},
83+
"span_id": {
84+
"kind": "string",
85+
"identifier": true
86+
},
87+
"flags": {
88+
"kind": "integer",
89+
"hidden": true
90+
},
91+
"scope_name": {
92+
"kind": "string",
93+
"identifier": true
94+
},
95+
"service.name": {
96+
"kind": "string",
97+
"identifier": true
98+
},
99+
"host.name": {
100+
"kind": "string",
101+
"identifier": true,
102+
"hidden": true
103+
},
104+
"deployment.environment": {
105+
"kind": "string",
106+
"identifier": true,
107+
"hidden": true
108+
}
109+
},
110+
"sample": [
111+
{
112+
"line": "{\"timestamp_ns\":\"1700000000000000000\",\"severity\":\"INFO\",\"severity_number\":9,\"body\":\"Application started successfully\",\"trace_id\":\"abc123\",\"span_id\":\"def456\",\"scope_name\":\"app.main\",\"service.name\":\"my-service\"}"
113+
}
114+
]
115+
}
116+
}

‎src/scripts/otel_collector_log-converter.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
# Check that jq is installed and return a nice message.
4+
if ! command -v jq > /dev/null 2>&1; then
5+
echo "error: otel_collector_log support requires 'jq' to be installed" >&2
6+
exit 1
7+
fi
8+
9+
# Validate input argument
10+
if [[ -z "$2" ]]; then
11+
echo "error: missing input file argument" >&2
12+
exit 1
13+
fi
14+
15+
# Validate input file exists
16+
if [[ ! -f "$2" ]]; then
17+
echo "error: file not found: $2" >&2
18+
exit 1
19+
fi
20+
21+
# We want jq output to come in UTC
22+
export TZ=UTC
23+
24+
# Convert OTEL Collector File Exporter JSON to one-record-per-line format
25+
# Input: JSON Lines where each line contains batched log records under resourceLogs
26+
# Output: JSON Lines with one log record per line, flattened with resource attributes
27+
exec jq -c '
28+
.resourceLogs[] |
29+
(.resource.attributes // [] | map({(.key): (.value | to_entries[0] | .value)}) | add // {}) as $resAttrs |
30+
.scopeLogs[] |
31+
(.scope.name // "") as $scope |
32+
.logRecords[] |
33+
{
34+
timestamp_ns: .timeUnixNano,
35+
observed_ns: (.observedTimeUnixNano // ""),
36+
severity_number: (.severityNumber // 0),
37+
severity: (.severityText // "UNSPECIFIED"),
38+
body: (.body.stringValue // (.body | tostring)),
39+
trace_id: (.traceId // ""),
40+
span_id: (.spanId // ""),
41+
flags: (.flags // 0),
42+
scope_name: $scope
43+
} + $resAttrs
44+
' -- "$2"

‎src/scripts/scripts.am‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ BUILTIN_LNAVSCRIPTS = \
2323
BUILTIN_SHSCRIPTS = \
2424
$(srcdir)/scripts/com.vmware.btresolver.py \
2525
$(srcdir)/scripts/dump-pid.sh \
26+
$(srcdir)/scripts/otel_collector_log-converter.sh \
2627
$(srcdir)/scripts/pcap_log-converter.sh \
2728
$(srcdir)/scripts/zookeeper.sql \
2829
$()

‎test/Makefile.am‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ dist_noinst_DATA = \
392392
logfile_mysql_slow.0 \
393393
logfile_nested_json.json \
394394
logfile_nextcloud.0 \
395+
logfile_otel_collector.jsonl \
395396
logfile_openam.0 \
396397
logfile_partitions.0 \
397398
logfile_pino.0 \

‎test/logfile_otel_collector.jsonl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"payment-service"}},{"key":"host.name","value":{"stringValue":"prod-01"}}]},"scopeLogs":[{"scope":{"name":"com.example.payment"},"logRecords":[{"timeUnixNano":"1700000000000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Processing payment request"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c13"},{"timeUnixNano":"1700000001000000000","severityNumber":13,"severityText":"WARN","body":{"stringValue":"Payment gateway slow response"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c14"},{"timeUnixNano":"1700000002000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Payment completed successfully"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c15"}]}]}]}
2+
{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"order-service"}},{"key":"host.name","value":{"stringValue":"prod-02"}}]},"scopeLogs":[{"scope":{"name":"com.example.order"},"logRecords":[{"timeUnixNano":"1700000003000000000","severityNumber":17,"severityText":"ERROR","body":{"stringValue":"Failed to update order status: database connection timeout"},"traceId":"6c9bb6b3e3d983f9432d048419e7ae03","spanId":"162692cf4dc66d24"},{"timeUnixNano":"1700000004000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Retrying database connection"},"traceId":"6c9bb6b3e3d983f9432d048419e7ae03","spanId":"162692cf4dc66d25"}]}]}]}
3+
{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"severity-test"}},{"key":"host.name","value":{"stringValue":"test-01"}}]},"scopeLogs":[{"scope":{"name":"com.example.test"},"logRecords":[{"timeUnixNano":"1700000005000000000","severityNumber":1,"severityText":"TRACE","body":{"stringValue":"Entering function processOrder()"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e35"},{"timeUnixNano":"1700000006000000000","severityNumber":5,"severityText":"DEBUG","body":{"stringValue":"Order validation passed, proceeding with payment"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e36"},{"timeUnixNano":"1700000007000000000","severityNumber":21,"severityText":"FATAL","body":{"stringValue":"System critical failure: out of memory"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e37"}]}]}]}
4+
{"resourceLogs":[{"resource":{"attributes":[]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1700000008000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Log with empty resource attributes and no scope name"}}]}]}]}
5+
{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"minimal-test"}}]},"scopeLogs":[{"scope":{"name":"minimal"},"logRecords":[{"timeUnixNano":"1700000009000000000","body":{"stringValue":"Log with missing severity fields - should default to UNSPECIFIED"}}]}]}]}
6+
{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"env-test"}},{"key":"host.name","value":{"stringValue":"prod-server"}},{"key":"deployment.environment","value":{"stringValue":"production"}}]},"scopeLogs":[{"scope":{"name":"com.example.deploy"},"logRecords":[{"timeUnixNano":"1700000010000000000","observedTimeUnixNano":"1700000010100000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Production deployment completed"},"traceId":"8e1dd8d505f1a50b654f060631090c25","spanId":"384814e06fe88f46","flags":1}]}]}]}

0 commit comments

Comments
 (0)