Skip to content

Commit d520a84

Browse files
TaeyeongKwakTaeyeongKwak
authored
feat(log): add nested log format support for logger plugins (#12697)
1 parent 614c44c commit d520a84

35 files changed

+509
-123
lines changed

‎apisix/utils/log-util.lua‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ local expr = require("resty.expr.v1")
2020
local content_decode = require("apisix.utils.content-decode")
2121
local ngx = ngx
2222
local pairs = pairs
23+
local type = type
2324
local ngx_now = ngx.now
2425
local ngx_header = ngx.header
2526
local os_date = os.date
@@ -32,6 +33,7 @@ local is_http = ngx.config.subsystem == "http"
3233
local req_get_body_file = ngx.req.get_body_file
3334
local MAX_REQ_BODY = 524288 -- 512 KiB
3435
local MAX_RESP_BODY = 524288 -- 512 KiB
36+
local MAX_LOG_FORMAT_DEPTH = 5
3537
local io = io
3638

3739
local lru_log_format = core.lrucache.new({
@@ -69,22 +71,35 @@ local function get_request_body(max_bytes)
6971
end
7072

7173

72-
local function gen_log_format(format)
74+
local function do_gen_log_format(format, depth)
7375
local log_format = {}
7476
for k, var_name in pairs(format) do
75-
if var_name:byte(1, 1) == str_byte("$") then
77+
if type(var_name) == "table" then
78+
if depth >= MAX_LOG_FORMAT_DEPTH then
79+
core.log.warn("log_format nesting exceeds max depth ",
80+
MAX_LOG_FORMAT_DEPTH, ", truncating")
81+
log_format[k] = {false, {}}
82+
else
83+
local nested_format = do_gen_log_format(var_name, depth + 1)
84+
log_format[k] = {false, nested_format}
85+
end
86+
elseif type(var_name) == "string" and var_name:byte(1, 1) == str_byte("$") then
7687
log_format[k] = {true, var_name:sub(2)}
7788
else
7889
log_format[k] = {false, var_name}
7990
end
8091
end
92+
return log_format
93+
end
94+
95+
local function gen_log_format(format)
96+
local log_format = do_gen_log_format(format, 1)
8197
core.log.info("log_format: ", core.json.delay_encode(log_format))
8298
return log_format
8399
end
84100

85101

86-
local function get_custom_format_log(ctx, format, max_req_body_bytes)
87-
local log_format = lru_log_format(format or "", nil, gen_log_format, format)
102+
local function build_log_entry(ctx, log_format, max_req_body_bytes)
88103
local entry = core.table.new(0, core.table.nkeys(log_format))
89104
for k, var_attr in pairs(log_format) do
90105
if var_attr[1] then
@@ -100,10 +115,19 @@ local function get_custom_format_log(ctx, format, max_req_body_bytes)
100115
else
101116
entry[k] = ctx.var[var_attr[2]]
102117
end
118+
elseif type(var_attr[2]) == "table" then
119+
entry[k] = build_log_entry(ctx, var_attr[2], max_req_body_bytes)
103120
else
104121
entry[k] = var_attr[2]
105122
end
106123
end
124+
return entry
125+
end
126+
127+
128+
local function get_custom_format_log(ctx, format, max_req_body_bytes)
129+
local log_format = lru_log_format(format or "", nil, gen_log_format, format)
130+
local entry = build_log_entry(ctx, log_format, max_req_body_bytes)
107131

108132
local matched_route = ctx.matched_route and ctx.matched_route.value
109133
if matched_route then

‎docs/en/latest/plugins/clickhouse-logger.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic
4444
| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. |
4545
| name | string | False | "clickhouse logger" | | Unique identifier for the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. |
4646
| ssl_verify | boolean | False | true | [true,false] | When set to `true`, verifies SSL. |
47-
| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
47+
| log_format | object | False | | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
4848
| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
4949
| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
5050
| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. |
@@ -103,7 +103,7 @@ You can also set the format of the logs by configuring the Plugin metadata. The
103103

104104
| Name | Type | Required | Default | Description |
105105
| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
106-
| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
106+
| log_format | object | False | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
107107
| max_pending_entries | integer | False | | Maximum number of pending entries that can be buffered in batch processor before it starts dropping them. |
108108

109109
:::info IMPORTANT

‎docs/en/latest/plugins/elasticsearch-logger.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ The `elasticsearch-logger` Plugin pushes request and response logs in batches to
4242
| endpoint_addrs | array[string] | True | | Elasticsearch API endpoint addresses. If multiple endpoints are configured, they will be written randomly. |
4343
| field | object | True | | Elasticsearch `field` configuration. |
4444
| field.index | string | True | | Elasticsearch [_index field](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html#mapping-index-field). |
45-
| log_format | object | False | | Custom log format in key-value pairs in JSON format. Support [APISIX](../apisix-variable.md) or [NGINX variables](http://nginx.org/en/docs/varindex.html) in values. |
45+
| log_format | object | False | | Custom log format as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX variables](http://nginx.org/en/docs/varindex.html) can be referenced by prefixing with `$`. |
4646
| auth | array | False | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) configuration. |
4747
| auth.username | string | True | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) username. |
4848
| auth.password | string | True | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) password. |
@@ -61,7 +61,7 @@ This Plugin supports using batch processors to aggregate and process entries (lo
6161

6262
| Name | Type | Required | Default | Description |
6363
|------|------|----------|---------|-------------|
64-
| log_format | object | False | | Custom log format in key-value pairs in JSON format. Support [APISIX variables](../apisix-variable.md) and [NGINX variables](http://nginx.org/en/docs/varindex.html) in values. |
64+
| log_format | object | False | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
6565
| max_pending_entries | integer | False | | Maximum number of pending entries that can be buffered in batch processor before it starts dropping them. |
6666

6767
## Examples

‎docs/en/latest/plugins/file-logger.md‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ The `file-logger` Plugin is used to push log streams to a specific location.
4646
| Name | Type | Required | Description |
4747
| ---- | ------ | -------- | ------------- |
4848
| path | string | True | Log file path. |
49-
| log_format | object | False | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
49+
| log_format | object | False | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
5050
| include_req_body | boolean | False | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
5151
| include_req_body_expr | array | False | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
5252
| include_resp_body | boolean | False | When set to `true` includes the response body in the log file. |
@@ -103,7 +103,7 @@ You can also set the format of the logs by configuring the Plugin metadata. The
103103

104104
| Name | Type | Required | Default | Description |
105105
| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
106-
| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
106+
| log_format | object | False | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
107107

108108
The example below shows how you can configure through the Admin API:
109109

@@ -122,16 +122,23 @@ curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/file-logger -H "X-API-KE
122122
"log_format": {
123123
"host": "$host",
124124
"@timestamp": "$time_iso8601",
125-
"client_ip": "$remote_addr"
125+
"client_ip": "$remote_addr",
126+
"request": {
127+
"method": "$request_method",
128+
"uri": "$request_uri"
129+
},
130+
"response": {
131+
"status": "$status"
132+
}
126133
}
127134
}'
128135
```
129136

130137
With this configuration, your logs would be formatted as shown below:
131138

132139
```shell
133-
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
134-
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","route_id":"1"}
140+
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","request":{"method":"GET","uri":"/hello"},"response":{"status":200},"route_id":"1"}
141+
{"host":"localhost","@timestamp":"2020-09-23T19:05:05-04:00","client_ip":"127.0.0.1","request":{"method":"GET","uri":"/hello"},"response":{"status":200},"route_id":"1"}
135142
```
136143

137144
## Enable Plugin

‎docs/en/latest/plugins/google-cloud-logging.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ This plugin also allows to push logs as a batch to your Google Cloud Logging Ser
4848
| ssl_verify | False | true | When set to `true`, enables SSL verification as mentioned in [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake). |
4949
| resource | False | {"type": "global"} | Google monitor resource. See [MonitoredResource](https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource) for more details. |
5050
| log_id | False | apisix.apache.org%2Flogs | Google Cloud logging ID. See [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) for details. |
51-
| log_format | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
51+
| log_format | False | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
5252

5353
NOTE: `encrypt_fields = {"auth_config.private_key"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
5454

@@ -90,7 +90,7 @@ You can also set the format of the logs by configuring the Plugin metadata. The
9090

9191
| Name | Type | Required | Default | Description |
9292
| ---------- | ------ | -------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
93-
| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
93+
| log_format | object | False | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. |
9494
| max_pending_entries | integer | False | | Maximum number of pending entries that can be buffered in batch processor before it starts dropping them. |
9595

9696
:::info IMPORTANT
@@ -116,15 +116,17 @@ curl http://127.0.0.1:9180/apisix/admin/plugin_metadata/google-cloud-logging -H
116116
"log_format": {
117117
"host": "$host",
118118
"@timestamp": "$time_iso8601",
119-
"client_ip": "$remote_addr"
119+
"client_ip": "$remote_addr",
120+
"request": { "method": "$request_method", "uri": "$request_uri" },
121+
"response": { "status": "$status" }
120122
}
121123
}'
122124
```
123125

124126
With this configuration, your logs would be formatted as shown below:
125127

126128
```json
127-
{"partialSuccess":false,"entries":[{"jsonPayload":{"client_ip":"127.0.0.1","host":"localhost","@timestamp":"2023-01-09T14:47:25+08:00","route_id":"1"},"resource":{"type":"global"},"insertId":"942e81f60b9157f0d46bc9f5a8f0cc40","logName":"projects/apisix/logs/apisix.apache.org%2Flogs","timestamp":"2023-01-09T14:47:25+08:00","labels":{"source":"apache-apisix-google-cloud-logging"}}]}
129+
{"partialSuccess":false,"entries":[{"jsonPayload":{"host":"localhost","client_ip":"127.0.0.1","@timestamp":"2023-01-09T14:47:25+08:00","request":{"method":"GET","uri":"/hello"},"response":{"status":200},"route_id":"1"},"resource":{"type":"global"},"insertId":"942e81f60b9157f0d46bc9f5a8f0cc40","logName":"projects/apisix/logs/apisix.apache.org%2Flogs","timestamp":"2023-01-09T14:47:25+08:00","labels":{"source":"apache-apisix-google-cloud-logging"}}]}
128130
```
129131

130132
## Enable Plugin

0 commit comments

Comments
 (0)