Skip to content

Commit cb65793

Browse files
itssimonitssimon
authored
Add default masking for request and response body fields (#22)
* WIP * WIP * Add tests * Fix
1 parent b6e1c9c commit cb65793

File tree

4 files changed

+477
-153
lines changed

4 files changed

+477
-153
lines changed

‎src/main/java/io/apitally/common/RequestLogger.java‎

Lines changed: 140 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import java.util.Arrays;
99
import java.util.Deque;
1010
import java.util.List;
11-
import java.util.UUID;
1211
import java.util.concurrent.ConcurrentLinkedDeque;
1312
import java.util.concurrent.Executors;
1413
import java.util.concurrent.ScheduledExecutorService;
@@ -21,12 +20,14 @@
2120
import org.slf4j.Logger;
2221
import org.slf4j.LoggerFactory;
2322

23+
import com.fasterxml.jackson.databind.JsonNode;
2424
import com.fasterxml.jackson.databind.ObjectMapper;
2525
import com.fasterxml.jackson.databind.node.ObjectNode;
2626

2727
import io.apitally.common.dto.ExceptionDto;
2828
import io.apitally.common.dto.Header;
2929
import io.apitally.common.dto.Request;
30+
import io.apitally.common.dto.RequestLogItem;
3031
import io.apitally.common.dto.Response;
3132

3233
public class RequestLogger {
@@ -40,6 +41,7 @@ public class RequestLogger {
4041
private static final byte[] BODY_MASKED = "<masked>".getBytes(StandardCharsets.UTF_8);
4142
private static final String MASKED = "******";
4243
public static final List<String> ALLOWED_CONTENT_TYPES = Arrays.asList("application/json", "text/plain");
44+
private static final Pattern JSON_CONTENT_TYPE_PATTERN = Pattern.compile("\\bjson\\b", Pattern.CASE_INSENSITIVE);
4345
private static final List<String> EXCLUDE_PATH_PATTERNS = Arrays.asList(
4446
"/_?healthz?$",
4547
"/_?health[_-]?checks?$",
@@ -65,12 +67,21 @@ public class RequestLogger {
6567
"secret",
6668
"token",
6769
"cookie");
70+
private static final List<String> MASK_BODY_FIELD_PATTERNS = Arrays.asList(
71+
"password",
72+
"pwd",
73+
"token",
74+
"secret",
75+
"auth",
76+
"card[-_ ]?number",
77+
"ccv",
78+
"ssn");
6879
private static final int MAINTAIN_INTERVAL_SECONDS = 1;
6980

7081
private final RequestLoggingConfig config;
7182
private final ObjectMapper objectMapper;
7283
private final ReentrantLock lock;
73-
private final Deque<String> pendingWrites;
84+
private final Deque<RequestLogItem> pendingWrites;
7485
private final Deque<TempGzipFile> files;
7586
private TempGzipFile currentFile;
7687
private boolean enabled;
@@ -82,6 +93,7 @@ public class RequestLogger {
8293
private final List<Pattern> compiledUserAgentExcludePatterns;
8394
private final List<Pattern> compiledQueryParamMaskPatterns;
8495
private final List<Pattern> compiledHeaderMaskPatterns;
96+
private final List<Pattern> compiledBodyFieldMaskPatterns;
8597

8698
public RequestLogger(RequestLoggingConfig config) {
8799
this.config = config;
@@ -96,6 +108,8 @@ public RequestLogger(RequestLoggingConfig config) {
96108
this.compiledQueryParamMaskPatterns = compilePatterns(MASK_QUERY_PARAM_PATTERNS,
97109
config.getQueryParamMaskPatterns());
98110
this.compiledHeaderMaskPatterns = compilePatterns(MASK_HEADER_PATTERNS, config.getHeaderMaskPatterns());
111+
this.compiledBodyFieldMaskPatterns = compilePatterns(MASK_BODY_FIELD_PATTERNS,
112+
config.getBodyFieldMaskPatterns());
99113

100114
if (enabled) {
101115
startMaintenance();
@@ -131,80 +145,27 @@ public void logRequest(Request request, Response response, Exception exception)
131145

132146
try {
133147
String userAgent = findHeader(request.getHeaders(), "user-agent");
134-
if (shouldExcludePath(request.getPath()) || shouldExcludeUserAgent(userAgent)
135-
|| (config.getCallbacks() != null && config.getCallbacks().shouldExclude(request, response))) {
148+
if (shouldExcludePath(request.getPath()) || shouldExcludeUserAgent(userAgent)) {
136149
return;
137150
}
138-
139-
// Process query params and URL
140-
if (request.getUrl() != null) {
141-
try {
142-
URL url = new URL(request.getUrl());
143-
String query = url.getQuery();
144-
if (!config.isQueryParamsIncluded()) {
145-
query = null;
146-
} else if (query != null) {
147-
query = maskQueryParams(query);
148-
}
149-
request.setUrl(new java.net.URL(url.getProtocol(), url.getHost(), url.getPort(),
150-
url.getPath() + (query != null ? "?" + query : "")).toString());
151-
} catch (MalformedURLException e) {
152-
return;
153-
}
151+
if (config.getCallbacks() != null && config.getCallbacks().shouldExclude(request, response)) {
152+
return;
154153
}
155154

156-
// Process request body
157155
if (!config.isRequestBodyIncluded() || !hasSupportedContentType(request.getHeaders())) {
158156
request.setBody(null);
159-
} else if (request.getBody() != null) {
160-
if (request.getBody().length > MAX_BODY_SIZE) {
161-
request.setBody(BODY_TOO_LARGE);
162-
} else if (config.getCallbacks() != null) {
163-
byte[] maskedBody = config.getCallbacks().maskRequestBody(request);
164-
request.setBody(maskedBody != null ? maskedBody : BODY_MASKED);
165-
if (request.getBody().length > MAX_BODY_SIZE) {
166-
request.setBody(BODY_TOO_LARGE);
167-
}
168-
}
169157
}
170-
171-
// Process response body
172158
if (!config.isResponseBodyIncluded() || !hasSupportedContentType(response.getHeaders())) {
173159
response.setBody(null);
174-
} else if (response.getBody() != null) {
175-
if (response.getBody().length > MAX_BODY_SIZE) {
176-
response.setBody(BODY_TOO_LARGE);
177-
} else if (config.getCallbacks() != null) {
178-
byte[] maskedBody = config.getCallbacks().maskResponseBody(request, response);
179-
response.setBody(maskedBody != null ? maskedBody : BODY_MASKED);
180-
if (response.getBody().length > MAX_BODY_SIZE) {
181-
response.setBody(BODY_TOO_LARGE);
182-
}
183-
}
184160
}
185161

186-
// Process headers
187-
request.setHeaders(
188-
config.isRequestHeadersIncluded()
189-
? maskHeaders(request.getHeaders()).toArray(new Header[0])
190-
: new Header[0]);
191-
response.setHeaders(
192-
config.isResponseHeadersIncluded()
193-
? maskHeaders(response.getHeaders()).toArray(new Header[0])
194-
: new Header[0]);
195-
196-
// Create log item
197-
ObjectNode item = objectMapper.createObjectNode();
198-
item.put("uuid", UUID.randomUUID().toString());
199-
item.set("request", skipEmptyValues(objectMapper.valueToTree(request)));
200-
item.set("response", skipEmptyValues(objectMapper.valueToTree(response)));
162+
ExceptionDto exceptionDto = null;
201163
if (exception != null && config.isExceptionIncluded()) {
202-
ExceptionDto exceptionDto = new ExceptionDto(exception);
203-
item.set("exception", objectMapper.valueToTree(exceptionDto));
164+
exceptionDto = new ExceptionDto(exception);
204165
}
205166

206-
String serializedItem = objectMapper.writeValueAsString(item);
207-
pendingWrites.add(serializedItem);
167+
RequestLogItem item = new RequestLogItem(request, response, exceptionDto);
168+
pendingWrites.add(item);
208169

209170
if (pendingWrites.size() > MAX_PENDING_WRITES) {
210171
pendingWrites.poll();
@@ -214,6 +175,74 @@ public void logRequest(Request request, Response response, Exception exception)
214175
}
215176
}
216177

178+
private void applyMasking(RequestLogItem item) {
179+
Request request = item.getRequest();
180+
Response response = item.getResponse();
181+
182+
if (request.getBody() != null) {
183+
// Apply user-provided masking callback for request body
184+
if (config.getCallbacks() != null) {
185+
byte[] maskedBody = config.getCallbacks().maskRequestBody(request);
186+
request.setBody(maskedBody != null ? maskedBody : BODY_MASKED);
187+
}
188+
189+
if (request.getBody().length > MAX_BODY_SIZE) {
190+
request.setBody(BODY_TOO_LARGE);
191+
}
192+
193+
// Mask request body fields (if JSON)
194+
if (!Arrays.equals(request.getBody(), BODY_TOO_LARGE) && !Arrays.equals(request.getBody(), BODY_MASKED)
195+
&& hasJsonContentType(request.getHeaders())) {
196+
request.setBody(maskJsonBody(request.getBody()));
197+
}
198+
}
199+
200+
if (response.getBody() != null) {
201+
// Apply user-provided masking callback for response body
202+
if (config.getCallbacks() != null) {
203+
byte[] maskedBody = config.getCallbacks().maskResponseBody(request, response);
204+
response.setBody(maskedBody != null ? maskedBody : BODY_MASKED);
205+
}
206+
207+
if (response.getBody().length > MAX_BODY_SIZE) {
208+
response.setBody(BODY_TOO_LARGE);
209+
}
210+
211+
// Mask response body fields (if JSON)
212+
if (!Arrays.equals(response.getBody(), BODY_TOO_LARGE) && !Arrays.equals(response.getBody(), BODY_MASKED)
213+
&& hasJsonContentType(response.getHeaders())) {
214+
response.setBody(maskJsonBody(response.getBody()));
215+
}
216+
}
217+
218+
// Process headers
219+
request.setHeaders(
220+
config.isRequestHeadersIncluded()
221+
? maskHeaders(request.getHeaders()).toArray(new Header[0])
222+
: new Header[0]);
223+
response.setHeaders(
224+
config.isResponseHeadersIncluded()
225+
? maskHeaders(response.getHeaders()).toArray(new Header[0])
226+
: new Header[0]);
227+
228+
// Process query params and URL
229+
if (request.getUrl() != null) {
230+
try {
231+
URL url = new URL(request.getUrl());
232+
String query = url.getQuery();
233+
if (!config.isQueryParamsIncluded()) {
234+
query = null;
235+
} else if (query != null) {
236+
query = maskQueryParams(query);
237+
}
238+
request.setUrl(new java.net.URL(url.getProtocol(), url.getHost(), url.getPort(),
239+
url.getPath() + (query != null ? "?" + query : "")).toString());
240+
} catch (MalformedURLException e) {
241+
// Keep original URL if malformed
242+
}
243+
}
244+
}
245+
217246
public void writeToFile() throws IOException {
218247
if (!enabled || pendingWrites.isEmpty()) {
219248
return;
@@ -223,9 +252,20 @@ public void writeToFile() throws IOException {
223252
if (currentFile == null) {
224253
currentFile = new TempGzipFile();
225254
}
226-
String item;
255+
RequestLogItem item;
227256
while ((item = pendingWrites.poll()) != null) {
228-
currentFile.writeLine(item.getBytes(StandardCharsets.UTF_8));
257+
applyMasking(item);
258+
259+
ObjectNode itemNode = objectMapper.createObjectNode();
260+
itemNode.put("uuid", item.getUuid());
261+
itemNode.set("request", skipEmptyValues(objectMapper.valueToTree(item.getRequest())));
262+
itemNode.set("response", skipEmptyValues(objectMapper.valueToTree(item.getResponse())));
263+
if (item.getException() != null) {
264+
itemNode.set("exception", objectMapper.valueToTree(item.getException()));
265+
}
266+
267+
String serializedItem = objectMapper.writeValueAsString(itemNode);
268+
currentFile.writeLine(serializedItem.getBytes(StandardCharsets.UTF_8));
229269
}
230270
} finally {
231271
lock.unlock();
@@ -344,6 +384,11 @@ private boolean shouldMaskHeader(String name) {
344384
.anyMatch(p -> p.matcher(name).find());
345385
}
346386

387+
private boolean shouldMaskBodyField(String name) {
388+
return compiledBodyFieldMaskPatterns.stream()
389+
.anyMatch(p -> p.matcher(name).find());
390+
}
391+
347392
private String maskQueryParams(String query) {
348393
if (query == null || query.isEmpty()) {
349394
return query;
@@ -371,12 +416,43 @@ private List<Header> maskHeaders(Header[] headers) {
371416
.collect(Collectors.toList());
372417
}
373418

419+
private byte[] maskJsonBody(byte[] body) {
420+
try {
421+
String json = new String(body, StandardCharsets.UTF_8);
422+
JsonNode node = objectMapper.readTree(json);
423+
maskJsonNode(node);
424+
return objectMapper.writeValueAsString(node).getBytes(StandardCharsets.UTF_8);
425+
} catch (Exception e) {
426+
return body;
427+
}
428+
}
429+
430+
private void maskJsonNode(JsonNode node) {
431+
if (node.isObject()) {
432+
ObjectNode objectNode = (ObjectNode) node;
433+
objectNode.fields().forEachRemaining(entry -> {
434+
if (entry.getValue().isTextual() && shouldMaskBodyField(entry.getKey())) {
435+
objectNode.put(entry.getKey(), MASKED);
436+
} else {
437+
maskJsonNode(entry.getValue());
438+
}
439+
});
440+
} else if (node.isArray()) {
441+
node.forEach(this::maskJsonNode);
442+
}
443+
}
444+
374445
private boolean hasSupportedContentType(Header[] headers) {
375446
String contentType = findHeader(headers, "content-type");
376447
return contentType != null && ALLOWED_CONTENT_TYPES.stream()
377448
.anyMatch(contentType::startsWith);
378449
}
379450

451+
private boolean hasJsonContentType(Header[] headers) {
452+
String contentType = findHeader(headers, "content-type");
453+
return contentType != null && JSON_CONTENT_TYPE_PATTERN.matcher(contentType).find();
454+
}
455+
380456
private String findHeader(Header[] headers, String name) {
381457
return Arrays.stream(headers)
382458
.filter(h -> h.getName().toLowerCase().equals(name))

‎src/main/java/io/apitally/common/RequestLoggingConfig.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class RequestLoggingConfig {
1313
private boolean exceptionIncluded = true;
1414
private List<String> queryParamMaskPatterns = new ArrayList<>();
1515
private List<String> headerMaskPatterns = new ArrayList<>();
16+
private List<String> bodyFieldMaskPatterns = new ArrayList<>();
1617
private List<String> pathExcludePatterns = new ArrayList<>();
1718
private RequestLoggingCallbacks callbacks;
1819

@@ -88,6 +89,14 @@ public void setHeaderMaskPatterns(List<String> headerMaskPatterns) {
8889
this.headerMaskPatterns = headerMaskPatterns;
8990
}
9091

92+
public List<String> getBodyFieldMaskPatterns() {
93+
return bodyFieldMaskPatterns;
94+
}
95+
96+
public void setBodyFieldMaskPatterns(List<String> bodyFieldMaskPatterns) {
97+
this.bodyFieldMaskPatterns = bodyFieldMaskPatterns;
98+
}
99+
91100
public List<String> getPathExcludePatterns() {
92101
return pathExcludePatterns;
93102
}

‎src/main/java/io/apitally/common/dto/RequestLogItem.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package io.apitally.common.dto;
2+
3+
import java.util.UUID;
4+
5+
import com.fasterxml.jackson.annotation.JsonProperty;
6+
7+
public class RequestLogItem extends BaseDto {
8+
private final String uuid;
9+
private final Request request;
10+
private final Response response;
11+
private final ExceptionDto exception;
12+
13+
public RequestLogItem(Request request, Response response, ExceptionDto exception) {
14+
this.uuid = UUID.randomUUID().toString();
15+
this.request = request;
16+
this.response = response;
17+
this.exception = exception;
18+
}
19+
20+
@JsonProperty("uuid")
21+
public String getUuid() {
22+
return uuid;
23+
}
24+
25+
@JsonProperty("request")
26+
public Request getRequest() {
27+
return request;
28+
}
29+
30+
@JsonProperty("response")
31+
public Response getResponse() {
32+
return response;
33+
}
34+
35+
@JsonProperty("exception")
36+
public ExceptionDto getException() {
37+
return exception;
38+
}
39+
}

0 commit comments

Comments
 (0)