Skip to content

Commit 8bdfdd0

Browse files
caroline-ttdcaroline-ttd
authored
Validate fields in audit log (#472)
* Validate fields in audit log * Add trace id validation * Update unit tests * Address comments * Update uid instance id validation for filtering out secrets and sql * Rename some variables * Change trace id validation logic same as uid instance id * Address comments
1 parent baaa360 commit 8bdfdd0

File tree

2 files changed

+543
-10
lines changed

2 files changed

+543
-10
lines changed

‎src/main/java/com/uid2/shared/audit/Audit.java‎

Lines changed: 154 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
11
package com.uid2.shared.audit;
22

3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
35
import io.vertx.core.MultiMap;
46
import io.vertx.core.buffer.Buffer;
57
import io.vertx.core.json.Json;
68
import io.vertx.core.json.JsonArray;
79
import io.vertx.core.json.JsonObject;
810
import io.vertx.ext.web.RequestBody;
911
import io.vertx.ext.web.RoutingContext;
12+
import lombok.Getter;
1013
import org.slf4j.Logger;
1114
import org.slf4j.LoggerFactory;
1215
import io.vertx.core.http.HttpServerRequest;
1316
import io.vertx.core.http.HttpServerResponse;
1417
import java.time.Instant;
1518
import java.util.*;
19+
import java.util.regex.Matcher;
20+
import java.util.regex.Pattern;
1621

1722
public class Audit {
1823

@@ -30,6 +35,9 @@ static class AuditRecord {
3035
private final JsonObject queryParams;
3136
private final String uidInstanceId;
3237
private final String requestBody;
38+
private final StringBuilder toJsonValidationErrorMessageBuilder = new StringBuilder();
39+
@Getter
40+
private String toJsonValidationErrorMessage = "";
3341

3442
private AuditRecord(Builder builder) {
3543
this.timestamp = Instant.now();
@@ -53,20 +61,150 @@ public JsonObject toJson() {
5361
.put("source", source)
5462
.put("status", status)
5563
.put("method", method)
56-
.put("endpoint", endpoint)
57-
.put("trace_id", traceId)
58-
.put("uid_instance_id", uidInstanceId)
59-
.put("actor", actor);
60-
if (uidTraceId != null) {
64+
.put("endpoint", endpoint);
65+
66+
if (traceId != null && validateId(traceId, "trace_id")) {
67+
json.put("trace_id", traceId);
68+
}
69+
70+
if (uidTraceId != null && validateId(uidTraceId, "uid_trace_id")) {
6171
json.put("uid_trace_id", uidTraceId);
6272
}
73+
74+
if (uidInstanceId != null && validateId(uidInstanceId, "uid_instance_id")) {
75+
json.put("uid_instance_id", uidInstanceId);
76+
}
6377
actor.put("id", this.getLogIdentifier(json));
64-
json.put("actor", actor);
65-
if (queryParams != null) json.put("query_params", queryParams);
66-
if (requestBody != null) json.put("request_body", requestBody);
78+
if (validateJsonObjectParams(actor, "actor")) {
79+
json.put("actor", actor);
80+
}
81+
if (queryParams != null && validateJsonObjectParams(queryParams, "query_params")) json.put("query_params", queryParams);
82+
if (requestBody != null) {
83+
String sanitizedRequestBody = sanitizeRequestBody(requestBody);
84+
if (!sanitizedRequestBody.isEmpty()) {
85+
json.put("request_body", sanitizedRequestBody);
86+
}
87+
}
88+
toJsonValidationErrorMessage = toJsonValidationErrorMessageBuilder.isEmpty()? "" : "Audit log failure: " + toJsonValidationErrorMessageBuilder.toString();
6789
return json;
6890
}
6991

92+
private boolean validateJsonObjectParams(JsonObject jsonObject, String propertyName) {
93+
Set<String> keysToRemove = new HashSet<>();
94+
95+
for (String key : jsonObject.fieldNames()) {
96+
String val = jsonObject.getString(key);
97+
98+
boolean containsNoSecret = validateNoSecrets(key, propertyName) && validateNoSecrets(val, propertyName);
99+
boolean containsNoSQL = validateNoSQL(key, propertyName) && validateNoSQL(val, propertyName);
100+
101+
if (!(containsNoSecret && containsNoSQL)) {
102+
keysToRemove.add(key);
103+
}
104+
105+
int parameter_max_length = 1000;
106+
if (val != null && val.length() > parameter_max_length) {
107+
val = val.substring(0, parameter_max_length);
108+
toJsonValidationErrorMessageBuilder.append(String.format(
109+
"The %s is too long in the audit log: %s. ", propertyName, key));
110+
}
111+
112+
jsonObject.put(key, val);
113+
}
114+
115+
for (String key : keysToRemove) {
116+
jsonObject.remove(key);
117+
}
118+
119+
return !jsonObject.isEmpty();
120+
}
121+
122+
private boolean validateJsonArrayParams(JsonArray jsonArray, String propertyName) {
123+
JsonArray newJsonArray = new JsonArray();
124+
125+
for (Object object : jsonArray) {
126+
if (object instanceof JsonObject) {
127+
if (validateJsonObjectParams((JsonObject)object, propertyName)) {
128+
newJsonArray.add(object);
129+
}
130+
} else {
131+
toJsonValidationErrorMessageBuilder.append("The request body is a JSON array, but one of its elements is not a JSON object.");
132+
}
133+
}
134+
135+
jsonArray.clear();
136+
jsonArray.addAll(newJsonArray);
137+
138+
return !jsonArray.isEmpty();
139+
}
140+
141+
private String sanitizeRequestBody(String requestBody) {
142+
ObjectMapper mapper = new ObjectMapper();
143+
String sanitizedRequestBody = "";
144+
145+
try {
146+
JsonNode root = mapper.readTree(requestBody);
147+
148+
if (root.isObject()) {
149+
JsonObject jsonObject = new JsonObject(mapper.writeValueAsString(root));
150+
if (validateJsonObjectParams(jsonObject, "request_body")) sanitizedRequestBody = jsonObject.toString();
151+
} else if (root.isArray()) {
152+
JsonArray jsonArray = new JsonArray(mapper.writeValueAsString(root));
153+
if (validateJsonArrayParams(jsonArray, "request_body")) sanitizedRequestBody = jsonArray.toString();
154+
} else {
155+
toJsonValidationErrorMessageBuilder.append("The request body of audit log is not a JSON object or array. ");
156+
}
157+
} catch (Exception e) {
158+
toJsonValidationErrorMessageBuilder.append("The request body of audit log is Invalid JSON: ").append(e.getMessage());
159+
160+
}
161+
162+
int request_body_max_length = 10000;
163+
if (sanitizedRequestBody.length() > request_body_max_length) {
164+
sanitizedRequestBody = sanitizedRequestBody.substring(0, request_body_max_length);
165+
toJsonValidationErrorMessageBuilder.append("Request body is too long in the audit log: %s. ");
166+
}
167+
return sanitizedRequestBody;
168+
}
169+
170+
private boolean validateNoSecrets(String fieldValue, String propertyName) {
171+
if (fieldValue == null || fieldValue.isEmpty()) {
172+
return true;
173+
}
174+
Pattern uid2_key_pattern = Pattern.compile("(UID2|EUID)-[A-Za-z]-[A-Za-z]-[A-Za-z0-9_-]+");
175+
Matcher matcher = uid2_key_pattern.matcher(fieldValue);
176+
if(matcher.find()) {
177+
toJsonValidationErrorMessageBuilder.append(String.format("Secret found in the audit log: %s. ", propertyName));
178+
return false;
179+
} else {
180+
return true;
181+
}
182+
}
183+
184+
private boolean validateNoSQL(String fieldValue, String propertyName) {
185+
if (fieldValue == null || fieldValue.isEmpty()) {
186+
return true;
187+
}
188+
Pattern sql_injection_pattern = Pattern.compile(
189+
"(?i)(\\bselect\\b\\s+.+\\s+\\bfrom\\b|\\bunion\\b\\s+\\bselect\\b|\\binsert\\b\\s+\\binto\\b|\\bdrop\\b\\s+\\btable\\b|--|#|\\bor\\b|\\band\\b|\\blike\\b|\\bin\\b\\s*\\(|;)"
190+
);
191+
if(sql_injection_pattern.matcher(fieldValue).find()) {
192+
toJsonValidationErrorMessageBuilder.append(String.format("SQL injection found in the audit log: %s. ", propertyName));
193+
return false;
194+
} else {
195+
return true;
196+
}
197+
}
198+
199+
private boolean validateId(String uidInstanceId, String propertyName) {
200+
if(uidInstanceId.length() < 100 && validateNoSecrets(uidInstanceId, propertyName) && validateNoSQL(uidInstanceId, propertyName) ) {
201+
return true;
202+
} else {
203+
toJsonValidationErrorMessageBuilder.append(String.format("Malformed %s found in the audit log. ", propertyName));
204+
return false;
205+
}
206+
}
207+
70208
private String getLogIdentifier(JsonObject logObject) {
71209
JsonObject actor = logObject.getJsonObject("actor");
72210
String email = (actor != null) ? actor.getString("email") : null;
@@ -130,6 +268,7 @@ public Audit (String source) {
130268
public static final String UID_TRACE_ID_HEADER = "UID-Trace-Id";
131269
public static final String UID_INSTANCE_ID_HEADER = "UID-Instance-Id";
132270
private static final Logger LOGGER = LoggerFactory.getLogger(Audit.class);
271+
private static final String UNKNOWN_ID = "unknown";
133272

134273
private static Set<String> flattenToDotNotation(JsonObject json, String parentKey) {
135274
Set<String> keys = new HashSet<>();
@@ -240,7 +379,7 @@ private String filterJsonArrayBody(JsonArray bodyJson, Set<String> allowedKeys)
240379
}
241380

242381
private String defaultIfNull(String s) {
243-
return s != null ? s : "unknown";
382+
return s != null ? s : UNKNOWN_ID;
244383
}
245384

246385
private String defaultIfNull(String s, String defaultValue) {
@@ -301,7 +440,12 @@ public void log(RoutingContext ctx, AuditParams params) {
301440
}
302441

303442
AuditRecord auditRecord = builder.build();
304-
LOGGER.info(auditRecord.toString());
443+
String auditRecordString = auditRecord.toString();
444+
if (!auditRecord.getToJsonValidationErrorMessage().isEmpty()) {
445+
LOGGER.error(auditRecord.getToJsonValidationErrorMessage() + auditRecordString);
446+
}
447+
LOGGER.info(auditRecordString);
448+
305449
} catch (Exception e) {
306450
LOGGER.warn("Failed to log audit record", e);
307451
}

0 commit comments

Comments
 (0)