Skip to content

Commit ea270ae

Browse files
TheTeaCatTheTeaCat
committed
Implement & doc ONLY_LOG_JSON_MAX_CONTENT_LENGTH env var
1 parent c7fdf66 commit ea270ae

File tree

4 files changed

+154
-83
lines changed

4 files changed

+154
-83
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ POC for a FireTail Kubernetes Sensor.
1212
| `BPF_EXPRESSION` | ❌ | `tcp and (port 80 or port 443)` | The BPF filter used by the sensor. See docs for syntax info: https://www.tcpdump.org/manpages/pcap-filter.7.html |
1313
| `DISABLE_SERVICE_IP_FILTERING` | ❌ | `true` | Disables polling Kubernetes for the IP addresses of services & subsequently ignoring all requests captured that aren't made to one of those IPs. |
1414
| `ENABLE_ONLY_LOG_JSON` | ❌ | `true` | Enables only logging requests where the content-type implies the payload should be JSON, or the payload is valid JSON regardless of the content-type. |
15+
| `ONLY_LOG_JSON_MAX_CONTENT_LENGTH` | ❌ | `1048576` | When `ENABLE_ONLY_LOG_JSON` is `true`, the sensor will only read request or response bodies to check if they're valid JSON if their length is less than `ONLY_LOG_JSON_MAX_CONTENT_LENGTH` bytes. |
1516
| `FIRETAIL_API_URL` | ❌ | `https://api.logging.eu-west-1.prod.firetail.app/logs/bulk` | The API url the sensor will send logs to. Defaults to the EU region production environment. |
1617
| `FIRETAIL_KUBERNETES_SENSOR_DEV_MODE` | ❌ | `true` | Enables debug logging when set to `true`, and reduces the max age of a log in a batch to be sent to FireTail. |
1718
| `FIRETAIL_KUBERNETES_SENSOR_DEV_SERVER_ENABLED` | ❌ | `true` | Enables a demo web server when set to `true`; useful for sending test requests to. |

‎src/is_json.go‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"strings"
1010
)
1111

12-
func isJson(req_and_resp *httpRequestAndResponse) bool {
13-
for _, headers := range []http.Header{req_and_resp.request.Header, req_and_resp.response.Header} {
12+
func isJson(reqAndResp *httpRequestAndResponse, maxContentLength int64) bool {
13+
for _, headers := range []http.Header{reqAndResp.request.Header, reqAndResp.response.Header} {
1414
contentTypeHeader := headers.Get("Content-Type")
1515
mediaType, _, err := mime.ParseMediaType(contentTypeHeader)
1616
if err == nil && mediaType == "application/json" {
@@ -21,20 +21,29 @@ func isJson(req_and_resp *httpRequestAndResponse) bool {
2121
}
2222
}
2323

24-
bodyBytes, err := io.ReadAll(req_and_resp.request.Body)
25-
req_and_resp.request.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes)))
26-
if err != nil {
27-
return false
28-
}
29-
var v map[string]interface{}
30-
if json.Unmarshal(bodyBytes, &v) == nil {
31-
return true
24+
if reqAndResp.request.ContentLength <= maxContentLength {
25+
bodyBytes, err := io.ReadAll(reqAndResp.request.Body)
26+
reqAndResp.request.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes)))
27+
if err != nil {
28+
return false
29+
}
30+
var v map[string]interface{}
31+
if json.Unmarshal(bodyBytes, &v) == nil {
32+
return true
33+
}
3234
}
3335

34-
bodyBytes, err = io.ReadAll(req_and_resp.response.Body)
35-
req_and_resp.response.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes)))
36-
if err != nil {
37-
return false
36+
if reqAndResp.response.ContentLength <= maxContentLength {
37+
bodyBytes, err := io.ReadAll(reqAndResp.response.Body)
38+
reqAndResp.response.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes)))
39+
if err != nil {
40+
return false
41+
}
42+
var v map[string]interface{}
43+
if json.Unmarshal(bodyBytes, &v) == nil {
44+
return true
45+
}
3846
}
39-
return json.Unmarshal(bodyBytes, &v) == nil
47+
48+
return false
4049
}

‎src/is_json_test.go‎

Lines changed: 114 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -9,92 +9,139 @@ import (
99

1010
func TestIsJson(t *testing.T) {
1111
tests := []struct {
12-
name string
13-
reqContentType string
14-
reqBody string
15-
respContentType string
16-
respBody string
17-
expectedResult bool
12+
name string
13+
reqContentType string
14+
reqBody string
15+
respContentType string
16+
respBody string
17+
maxContentLength int64
18+
expectedResult bool
1819
}{
1920
{
20-
name: "Valid JSON in both request and response with correct content types",
21-
reqContentType: "application/json",
22-
reqBody: `{"key": "value"}`,
23-
respContentType: "application/json",
24-
respBody: `{"key": "value"}`,
25-
expectedResult: true,
21+
name: "Valid JSON in both request and response with correct content types",
22+
reqContentType: "application/json",
23+
reqBody: `{"key": "value"}`,
24+
respContentType: "application/json",
25+
respBody: `{"key": "value"}`,
26+
maxContentLength: 1024,
27+
expectedResult: true,
2628
},
2729
{
28-
name: "XML in request and response with correct Content-Type",
29-
reqContentType: "application/xml",
30-
reqBody: `<key>value</key>`,
31-
respContentType: "application/xml",
32-
respBody: `<key>value</key>`,
33-
expectedResult: false,
30+
name: "XML in request and response with correct Content-Type",
31+
reqContentType: "application/xml",
32+
reqBody: `<key>value</key>`,
33+
respContentType: "application/xml",
34+
respBody: `<key>value</key>`,
35+
maxContentLength: 1024,
36+
expectedResult: false,
3437
},
3538
{
36-
name: "XML in request with JSON in response",
37-
reqContentType: "application/xml",
38-
reqBody: `<key>value</key>`,
39-
respContentType: "application/json",
40-
respBody: `{"key": "value"}`,
41-
expectedResult: true,
39+
name: "XML in request with JSON in response",
40+
reqContentType: "application/xml",
41+
reqBody: `<key>value</key>`,
42+
respContentType: "application/json",
43+
respBody: `{"key": "value"}`,
44+
maxContentLength: 1024,
45+
expectedResult: true,
4246
},
4347
{
44-
name: "JSON in request with XML in response",
45-
reqContentType: "application/json",
46-
reqBody: `{"key": "value"}`,
47-
respContentType: "application/xml",
48-
respBody: `<key>value</key>`,
49-
expectedResult: true,
48+
name: "JSON in request with XML in response",
49+
reqContentType: "application/json",
50+
reqBody: `{"key": "value"}`,
51+
respContentType: "application/xml",
52+
respBody: `<key>value</key>`,
53+
maxContentLength: 1024,
54+
expectedResult: true,
5055
},
5156
{
52-
name: "Empty request and response bodies and headers",
53-
reqContentType: "",
54-
reqBody: "",
55-
respContentType: "",
56-
respBody: "",
57-
expectedResult: false,
57+
name: "Empty request and response bodies and headers",
58+
reqContentType: "",
59+
reqBody: "",
60+
respContentType: "",
61+
respBody: "",
62+
maxContentLength: 1024,
63+
expectedResult: false,
5864
},
5965
{
60-
name: "No content-type headers with valid JSON in request",
61-
reqContentType: "",
62-
reqBody: `{"key": "value"}`,
63-
respContentType: "",
64-
respBody: ``,
65-
expectedResult: true,
66+
name: "No content-type headers with valid JSON in request",
67+
reqContentType: "",
68+
reqBody: `{"key": "value"}`,
69+
respContentType: "",
70+
respBody: ``,
71+
maxContentLength: 1024,
72+
expectedResult: true,
6673
},
6774
{
68-
name: "No content-type headers with valid JSON in response",
69-
reqContentType: "",
70-
reqBody: ``,
71-
respContentType: "",
72-
respBody: `{"key": "value"}`,
73-
expectedResult: true,
75+
name: "No content-type headers with valid JSON in response",
76+
reqContentType: "",
77+
reqBody: ``,
78+
respContentType: "",
79+
respBody: `{"key": "value"}`,
80+
maxContentLength: 1024,
81+
expectedResult: true,
7482
},
7583
{
76-
name: "No content-type headers with invalid JSON in request",
77-
reqContentType: "",
78-
reqBody: `{"key": "value"`,
79-
respContentType: "",
80-
respBody: ``,
81-
expectedResult: false,
84+
name: "No content-type headers with invalid JSON in request",
85+
reqContentType: "",
86+
reqBody: `{"key": "value"`,
87+
respContentType: "",
88+
respBody: ``,
89+
maxContentLength: 1024,
90+
expectedResult: false,
8291
},
8392
{
84-
name: "No content-type headers with invalid JSON in response",
85-
reqContentType: "",
86-
reqBody: ``,
87-
respContentType: "",
88-
respBody: `{"key": "value"`,
89-
expectedResult: false,
93+
name: "No content-type headers with invalid JSON in response",
94+
reqContentType: "",
95+
reqBody: ``,
96+
respContentType: "",
97+
respBody: `{"key": "value"`,
98+
maxContentLength: 1024,
99+
expectedResult: false,
90100
},
91101
{
92-
name: "Content-type geo+json in request with invalid body",
93-
reqContentType: "application/geo+json",
94-
reqBody: ``,
95-
respContentType: "",
96-
respBody: ``,
97-
expectedResult: true,
102+
name: "Content-type geo+json in request with invalid body",
103+
reqContentType: "application/geo+json",
104+
reqBody: ``,
105+
respContentType: "",
106+
respBody: ``,
107+
maxContentLength: 1024,
108+
expectedResult: true,
109+
},
110+
{
111+
name: "No content-type headers with request payload longer than max length",
112+
reqContentType: "",
113+
reqBody: strings.Repeat("a", 1025),
114+
respContentType: "",
115+
respBody: ``,
116+
maxContentLength: 1024,
117+
expectedResult: false,
118+
},
119+
{
120+
name: "No content-type headers with response payload longer than max length",
121+
reqContentType: "",
122+
reqBody: ``,
123+
respContentType: "",
124+
respBody: strings.Repeat("a", 1025),
125+
maxContentLength: 1024,
126+
expectedResult: false,
127+
},
128+
{
129+
name: "No content-type headers with request payload longer than max length and response payload shorter",
130+
reqContentType: "",
131+
reqBody: strings.Repeat("a", 1025),
132+
respContentType: "",
133+
respBody: `{"key": "value"}`,
134+
maxContentLength: 1024,
135+
expectedResult: true,
136+
},
137+
{
138+
name: "No content-type headers with request payload shorter than max length and response payload longer",
139+
reqContentType: "",
140+
reqBody: `{"key": "value"}`,
141+
respContentType: "",
142+
respBody: strings.Repeat("a", 1025),
143+
maxContentLength: 1024,
144+
expectedResult: true,
98145
},
99146
}
100147

@@ -121,7 +168,7 @@ func TestIsJson(t *testing.T) {
121168
response: resp,
122169
}
123170

124-
result := isJson(&reqAndResp)
171+
result := isJson(&reqAndResp, tt.maxContentLength)
125172
if result != tt.expectedResult {
126173
t.Errorf("isJson() = %v, want %v", result, tt.expectedResult)
127174
}

‎src/main.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,21 @@ func main() {
5454
ipManager = newServiceIpManager()
5555
}
5656

57+
var maxContentLength int64
5758
onlyLogJson, _ := strconv.ParseBool(os.Getenv("ENABLE_ONLY_LOG_JSON"))
59+
if onlyLogJson {
60+
maxContentLengthStr, maxContentLengthSet := os.LookupEnv("ONLY_LOG_JSON_MAX_CONTENT_LENGTH")
61+
if !maxContentLengthSet {
62+
slog.Info("ONLY_LOG_JSON_MAX_CONTENT_LENGTH environment variable not set, using default: 1MiB")
63+
maxContentLength = 1048576 // 1MiB
64+
} else {
65+
maxContentLength, err = strconv.ParseInt(maxContentLengthStr, 10, 64)
66+
if err != nil {
67+
slog.Error("Failed to parse ONLY_LOG_JSON_MAX_CONTENT_LENGTH, Defaulting to 1MiB.", "Err", err.Error())
68+
maxContentLength = 1048576 // 1MiB
69+
}
70+
}
71+
}
5872

5973
requestAndResponseChannel := make(chan httpRequestAndResponse, 1)
6074
httpRequestStreamer := &httpRequestAndResponseStreamer{
@@ -93,7 +107,7 @@ func main() {
93107
)
94108
continue
95109
}
96-
if onlyLogJson && !isJson(&requestAndResponse) {
110+
if onlyLogJson && !isJson(&requestAndResponse, maxContentLength) {
97111
slog.Debug(
98112
"Ignoring non-JSON request:",
99113
"Src", requestAndResponse.src,

0 commit comments

Comments
 (0)