From 65e9307efa742c52737ae4c20d0b280af327ce8e Mon Sep 17 00:00:00 2001 From: takeokunn Date: Fri, 2 Jan 2026 23:52:33 +0900 Subject: [PATCH] feat(formats): add OpenTelemetry Collector log format and converter - Add otel_collector_log.json format definition for OpenTelemetry Collector - Add otel_collector_log-converter.sh script for flattening OTEL logs - Register new format and script in build system - Add test log file and update test Makefile for OTEL Collector logs - Improve error handling and argument validation in pcap_log-converter.sh --- src/formats/formats.am | 1 + src/formats/otel_collector_log.json | 116 ++++++++++++++++++++ src/scripts/otel_collector_log-converter.sh | 44 ++++++++ src/scripts/scripts.am | 1 + test/Makefile.am | 1 + test/logfile_otel_collector.jsonl | 6 + 6 files changed, 169 insertions(+) create mode 100644 src/formats/otel_collector_log.json create mode 100755 src/scripts/otel_collector_log-converter.sh create mode 100644 test/logfile_otel_collector.jsonl diff --git a/src/formats/formats.am b/src/formats/formats.am index 4c564ff4ef3..c2b7ec96b33 100644 --- a/src/formats/formats.am +++ b/src/formats/formats.am @@ -36,6 +36,7 @@ FORMAT_FILES = \ $(srcdir)/%reldir%/openam_log.json \ $(srcdir)/%reldir%/openamdb_log.json \ $(srcdir)/%reldir%/openstack_log.json \ + $(srcdir)/%reldir%/otel_collector_log.json \ $(srcdir)/%reldir%/otlp_python_log.json \ $(srcdir)/%reldir%/page_log.json \ $(srcdir)/%reldir%/pcap_log.json \ diff --git a/src/formats/otel_collector_log.json b/src/formats/otel_collector_log.json new file mode 100644 index 00000000000..88feac4f184 --- /dev/null +++ b/src/formats/otel_collector_log.json @@ -0,0 +1,116 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "otel_collector_log": { + "title": "OpenTelemetry Collector File Exporter", + "description": "Format for OpenTelemetry Collector file exporter JSON logs", + "url": "https://opentelemetry.io/docs/specs/otel/protocol/file-exporter/", + "file-type": "json", + "convert-to-local-time": true, + "converter": { + "header": { + "expr": { + "otel_collector": ":header REGEXP '.*\"resourceLogs\".*'" + }, + "size": 100 + }, + "command": "otel_collector_log-converter.sh" + }, + "line-format": [ + { + "field": "__timestamp__" + }, + " ", + { + "field": "__level__", + "text-transform": "uppercase", + "min-width": 5 + }, + " ", + { + "field": "service.name", + "default-value": "-", + "auto-width": true + }, + " ", + { + "field": "body" + }, + { + "field": "trace_id", + "prefix": " [trace:", + "suffix": "]", + "default-value": "" + } + ], + "level-field": "severity", + "level": { + "fatal": "FATAL", + "error": "ERROR", + "warning": "WARN", + "info": "INFO", + "debug": "DEBUG", + "trace": "TRACE" + }, + "timestamp-field": "timestamp_ns", + "timestamp-divisor": 1000000, + "body-field": "body", + "opid-field": "trace_id", + "hide-extra": true, + "value": { + "timestamp_ns": { + "kind": "string", + "hidden": true + }, + "observed_ns": { + "kind": "string", + "hidden": true + }, + "severity_number": { + "kind": "integer", + "identifier": true + }, + "severity": { + "kind": "string", + "identifier": true + }, + "body": { + "kind": "string" + }, + "trace_id": { + "kind": "string", + "identifier": true + }, + "span_id": { + "kind": "string", + "identifier": true + }, + "flags": { + "kind": "integer", + "hidden": true + }, + "scope_name": { + "kind": "string", + "identifier": true + }, + "service.name": { + "kind": "string", + "identifier": true + }, + "host.name": { + "kind": "string", + "identifier": true, + "hidden": true + }, + "deployment.environment": { + "kind": "string", + "identifier": true, + "hidden": true + } + }, + "sample": [ + { + "line": "{\"timestamp_ns\":\"1700000000000000000\",\"severity\":\"INFO\",\"severity_number\":9,\"body\":\"Application started successfully\",\"trace_id\":\"abc123\",\"span_id\":\"def456\",\"scope_name\":\"app.main\",\"service.name\":\"my-service\"}" + } + ] + } +} diff --git a/src/scripts/otel_collector_log-converter.sh b/src/scripts/otel_collector_log-converter.sh new file mode 100755 index 00000000000..44aaef17482 --- /dev/null +++ b/src/scripts/otel_collector_log-converter.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Check that jq is installed and return a nice message. +if ! command -v jq > /dev/null 2>&1; then + echo "error: otel_collector_log support requires 'jq' to be installed" >&2 + exit 1 +fi + +# Validate input argument +if [[ -z "$2" ]]; then + echo "error: missing input file argument" >&2 + exit 1 +fi + +# Validate input file exists +if [[ ! -f "$2" ]]; then + echo "error: file not found: $2" >&2 + exit 1 +fi + +# We want jq output to come in UTC +export TZ=UTC + +# Convert OTEL Collector File Exporter JSON to one-record-per-line format +# Input: JSON Lines where each line contains batched log records under resourceLogs +# Output: JSON Lines with one log record per line, flattened with resource attributes +exec jq -c ' + .resourceLogs[] | + (.resource.attributes // [] | map({(.key): (.value | to_entries[0] | .value)}) | add // {}) as $resAttrs | + .scopeLogs[] | + (.scope.name // "") as $scope | + .logRecords[] | + { + timestamp_ns: .timeUnixNano, + observed_ns: (.observedTimeUnixNano // ""), + severity_number: (.severityNumber // 0), + severity: (.severityText // "UNSPECIFIED"), + body: (.body.stringValue // (.body | tostring)), + trace_id: (.traceId // ""), + span_id: (.spanId // ""), + flags: (.flags // 0), + scope_name: $scope + } + $resAttrs +' -- "$2" diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index 91ade985a21..c9e153128b7 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -23,6 +23,7 @@ BUILTIN_LNAVSCRIPTS = \ BUILTIN_SHSCRIPTS = \ $(srcdir)/scripts/com.vmware.btresolver.py \ $(srcdir)/scripts/dump-pid.sh \ + $(srcdir)/scripts/otel_collector_log-converter.sh \ $(srcdir)/scripts/pcap_log-converter.sh \ $(srcdir)/scripts/zookeeper.sql \ $() diff --git a/test/Makefile.am b/test/Makefile.am index c30af7f96c9..0f7d2136573 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -392,6 +392,7 @@ dist_noinst_DATA = \ logfile_mysql_slow.0 \ logfile_nested_json.json \ logfile_nextcloud.0 \ + logfile_otel_collector.jsonl \ logfile_openam.0 \ logfile_partitions.0 \ logfile_pino.0 \ diff --git a/test/logfile_otel_collector.jsonl b/test/logfile_otel_collector.jsonl new file mode 100644 index 00000000000..c74d11e96da --- /dev/null +++ b/test/logfile_otel_collector.jsonl @@ -0,0 +1,6 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"payment-service"}},{"key":"host.name","value":{"stringValue":"prod-01"}}]},"scopeLogs":[{"scope":{"name":"com.example.payment"},"logRecords":[{"timeUnixNano":"1700000000000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Processing payment request"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c13"},{"timeUnixNano":"1700000001000000000","severityNumber":13,"severityText":"WARN","body":{"stringValue":"Payment gateway slow response"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c14"},{"timeUnixNano":"1700000002000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Payment completed successfully"},"traceId":"5b8aa5a2d2c872e8321cf37308d69df2","spanId":"051581bf3cb55c15"}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"order-service"}},{"key":"host.name","value":{"stringValue":"prod-02"}}]},"scopeLogs":[{"scope":{"name":"com.example.order"},"logRecords":[{"timeUnixNano":"1700000003000000000","severityNumber":17,"severityText":"ERROR","body":{"stringValue":"Failed to update order status: database connection timeout"},"traceId":"6c9bb6b3e3d983f9432d048419e7ae03","spanId":"162692cf4dc66d24"},{"timeUnixNano":"1700000004000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Retrying database connection"},"traceId":"6c9bb6b3e3d983f9432d048419e7ae03","spanId":"162692cf4dc66d25"}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"severity-test"}},{"key":"host.name","value":{"stringValue":"test-01"}}]},"scopeLogs":[{"scope":{"name":"com.example.test"},"logRecords":[{"timeUnixNano":"1700000005000000000","severityNumber":1,"severityText":"TRACE","body":{"stringValue":"Entering function processOrder()"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e35"},{"timeUnixNano":"1700000006000000000","severityNumber":5,"severityText":"DEBUG","body":{"stringValue":"Order validation passed, proceeding with payment"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e36"},{"timeUnixNano":"1700000007000000000","severityNumber":21,"severityText":"FATAL","body":{"stringValue":"System critical failure: out of memory"},"traceId":"7d0cc7c4f4e094fa543e059520f8bf14","spanId":"273703d05ed77e37"}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1700000008000000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Log with empty resource attributes and no scope name"}}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"minimal-test"}}]},"scopeLogs":[{"scope":{"name":"minimal"},"logRecords":[{"timeUnixNano":"1700000009000000000","body":{"stringValue":"Log with missing severity fields - should default to UNSPECIFIED"}}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"env-test"}},{"key":"host.name","value":{"stringValue":"prod-server"}},{"key":"deployment.environment","value":{"stringValue":"production"}}]},"scopeLogs":[{"scope":{"name":"com.example.deploy"},"logRecords":[{"timeUnixNano":"1700000010000000000","observedTimeUnixNano":"1700000010100000000","severityNumber":9,"severityText":"INFO","body":{"stringValue":"Production deployment completed"},"traceId":"8e1dd8d505f1a50b654f060631090c25","spanId":"384814e06fe88f46","flags":1}]}]}]}