Skip to content

Commit 856bbdd

Browse files
ygreeygree
committed
[AWS Payload Tagging] Fix the default redaction rules and the redaction rules extraction tool.
1 parent f49ac98 commit 856bbdd

File tree

2 files changed

+126
-58
lines changed

2 files changed

+126
-58
lines changed

‎dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java‎

Lines changed: 99 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -253,60 +253,120 @@ public final class ConfigDefaults {
253253

254254
public static final List<String> DEFAULT_CLOUD_COMMON_PAYLOAD_TAGGING =
255255
asList(
256-
// Sns
257-
"$.Attributes.KmsMasterKeyId",
258-
"$.Attributes.Token",
259-
// EventBridge (RedactionRulesExtractor.java for eventbridge-2015-10-07)
260-
"$.AuthParameters.OAuthParameters.OAuthHttpParameters.HeaderParameters[*].Value",
261-
"$.AuthParameters.OAuthParameters.OAuthHttpParameters.QueryStringParameters[*].Value",
262-
"$.AuthParameters.OAuthParameters.OAuthHttpParameters.BodyParameters[*].Value",
263-
"$.AuthParameters.InvocationHttpParameters.HeaderParameters[*].Value",
264-
"$.AuthParameters.InvocationHttpParameters.QueryStringParameters[*].Value",
265-
"$.AuthParameters.InvocationHttpParameters.BodyParameters[*].Value",
266-
"$.Targets[*].RedshiftDataParameters.Sql",
267-
"$.Targets[*].RedshiftDataParameters.Sqls",
268-
"$.Targets[*].AppSyncParameters.GraphQLOperation",
269256
// S3 (RedactionRulesExtractor.java for s3-2006-03-01)
270-
"$.SSEKMSKeyId",
271-
"$.SSEKMSEncryptionContext",
272-
"$.ServerSideEncryptionConfiguration.Rules[*].ApplyServerSideEncryptionByDefault.KMSMasterKeyID",
273-
"$.InventoryConfiguration.Destination.S3BucketDestination.Encryption.SSEKMS.KeyId");
257+
// ./services/s3/src/main/resources/codegen-resources/service-2.json
258+
"$.CopyObject.SSEKMSKeyId",
259+
"$.CopyObject.SSEKMSEncryptionContext",
260+
"$.CreateMultipartUpload.SSEKMSKeyId",
261+
"$.CreateMultipartUpload.SSEKMSEncryptionContext",
262+
"$.PutObject.SSEKMSKeyId",
263+
"$.PutObject.SSEKMSEncryptionContext",
264+
265+
// https://github.com/DataDog/dd-trace-js/blob/5658024a4a0c78476be6b04c13e821b6f9f86762/packages/dd-trace/src/payload-tagging/config/aws.json
266+
"$.Attributes.KmsMasterKeyId",
267+
"$.Attributes.Token");
274268

275269
public static final List<String> DEFAULT_CLOUD_REQUEST_PAYLOAD_TAGGING =
276270
asList(
277-
// Sns
271+
// EventBridge (RedactionRulesExtractor.java for eventbridge-2015-10-07)
272+
// ./services/eventbridge/src/main/resources/codegen-resources/service-2.json
273+
"$.CreateConnection.AuthParameters.BasicAuthParameters.Password",
274+
"$.CreateConnection.AuthParameters.OAuthParameters.ClientParameters.ClientSecret",
275+
"$.CreateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.HeaderParameters[*].Value",
276+
"$.CreateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.QueryStringParameters[*].Value",
277+
"$.CreateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.BodyParameters[*].Value",
278+
"$.CreateConnection.AuthParameters.ApiKeyAuthParameters.ApiKeyValue",
279+
"$.CreateConnection.AuthParameters.InvocationHttpParameters.HeaderParameters[*].Value",
280+
"$.CreateConnection.AuthParameters.InvocationHttpParameters.QueryStringParameters[*].Value",
281+
"$.CreateConnection.AuthParameters.InvocationHttpParameters.BodyParameters[*].Value",
282+
"$.PutTargets.Targets[*].RedshiftDataParameters.Sql",
283+
"$.PutTargets.Targets[*].RedshiftDataParameters.Sqls",
284+
"$.PutTargets.Targets[*].AppSyncParameters.GraphQLOperation",
285+
"$.UpdateConnection.AuthParameters.BasicAuthParameters.Password",
286+
"$.UpdateConnection.AuthParameters.OAuthParameters.ClientParameters.ClientSecret",
287+
"$.UpdateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.HeaderParameters[*].Value",
288+
"$.UpdateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.QueryStringParameters[*].Value",
289+
"$.UpdateConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.BodyParameters[*].Value",
290+
"$.UpdateConnection.AuthParameters.ApiKeyAuthParameters.ApiKeyValue",
291+
"$.UpdateConnection.AuthParameters.InvocationHttpParameters.HeaderParameters[*].Value",
292+
"$.UpdateConnection.AuthParameters.InvocationHttpParameters.QueryStringParameters[*].Value",
293+
"$.UpdateConnection.AuthParameters.InvocationHttpParameters.BodyParameters[*].Value",
294+
295+
// Sns (RedactionRulesExtractor.java for sns-2010-03-31)
296+
// ./services/sns/src/main/resources/codegen-resources/service-2.json
297+
"$.CheckIfPhoneNumberIsOptedOut.phoneNumber",
298+
"$.CreateSMSSandboxPhoneNumber.PhoneNumber",
299+
"$.DeleteSMSSandboxPhoneNumber.PhoneNumber",
300+
"$.OptInPhoneNumber.phoneNumber",
301+
"$.Publish.PhoneNumber",
302+
"$.VerifySMSSandboxPhoneNumber.PhoneNumber",
303+
304+
// S3 (RedactionRulesExtractor.java for s3-2006-03-01)
305+
// ./services/s3/src/main/resources/codegen-resources/service-2.json
306+
"$.CompleteMultipartUpload.SSECustomerKey",
307+
"$.CopyObject.SSECustomerKey",
308+
"$.CopyObject.CopySourceSSECustomerKey",
309+
"$.CreateMultipartUpload.SSECustomerKey",
310+
"$.GetObject.SSECustomerKey",
311+
"$.GetObjectAttributes.SSECustomerKey",
312+
"$.HeadObject.SSECustomerKey",
313+
"$.ListParts.SSECustomerKey",
314+
"$.PutBucketEncryption.ServerSideEncryptionConfiguration.Rules[*].ApplyServerSideEncryptionByDefault.KMSMasterKeyID",
315+
"$.PutBucketInventoryConfiguration.InventoryConfiguration.Destination.S3BucketDestination.Encryption.SSEKMS.KeyId",
316+
"$.PutObject.SSECustomerKey",
317+
"$.RestoreObject.RestoreRequest.OutputLocation.S3.Encryption.KMSKeyId",
318+
"$.SelectObjectContent.SSECustomerKey",
319+
"$.UploadPart.SSECustomerKey",
320+
"$.UploadPartCopy.SSECustomerKey",
321+
"$.UploadPartCopy.CopySourceSSECustomerKey",
322+
"$.WriteGetObjectResponse.SSEKMSKeyId",
323+
324+
// https://github.com/DataDog/dd-trace-js/blob/5658024a4a0c78476be6b04c13e821b6f9f86762/packages/dd-trace/src/payload-tagging/config/aws.json
278325
"$.Attributes.PlatformCredential",
279326
"$.Attributes.PlatformPrincipal",
280327
"$.AWSAccountId",
281328
"$.Endpoint",
282329
"$.Token",
283-
"$.OneTimePassword",
284-
// Sns (RedactionRulesExtractor.java for sns-2010-03-31)
285-
"$.phoneNumber",
286-
"$.PhoneNumber",
287-
// EventBridge (RedactionRulesExtractor.java for eventbridge-2015-10-07)
288-
"$.AuthParameters.BasicAuthParameters.Password",
289-
"$.AuthParameters.OAuthParameters.ClientParameters.ClientSecret",
290-
"$.AuthParameters.ApiKeyAuthParameters.ApiKeyValue",
291-
// S3 (RedactionRulesExtractor.java for s3-2006-03-01)
292-
"$.SSECustomerKey",
293-
"$.CopySourceSSECustomerKey",
294-
"$.RestoreRequest.OutputLocation.S3.Encryption.KMSKeyId");
330+
"$.OneTimePassword");
295331

296332
public static final List<String> DEFAULT_CLOUD_RESPONSE_PAYLOAD_TAGGING =
297333
asList(
298-
// Sns
334+
// EventBridge (RedactionRulesExtractor.java for eventbridge-2015-10-07)
335+
// ./services/eventbridge/src/main/resources/codegen-resources/service-2.json
336+
"$.DescribeConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.HeaderParameters[*].Value",
337+
"$.DescribeConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.QueryStringParameters[*].Value",
338+
"$.DescribeConnection.AuthParameters.OAuthParameters.OAuthHttpParameters.BodyParameters[*].Value",
339+
"$.DescribeConnection.AuthParameters.InvocationHttpParameters.HeaderParameters[*].Value",
340+
"$.DescribeConnection.AuthParameters.InvocationHttpParameters.QueryStringParameters[*].Value",
341+
"$.DescribeConnection.AuthParameters.InvocationHttpParameters.BodyParameters[*].Value",
342+
"$.ListTargetsByRule.Targets[*].RedshiftDataParameters.Sql",
343+
"$.ListTargetsByRule.Targets[*].RedshiftDataParameters.Sqls",
344+
"$.ListTargetsByRule.Targets[*].AppSyncParameters.GraphQLOperation",
345+
346+
// Sns (Generated by RedactionRulesExtractor.java for sns-2010-03-31)
347+
// ./services/sns/src/main/resources/codegen-resources/service-2.json
348+
"$.ListOriginationNumbers.PhoneNumbers[*].PhoneNumber",
349+
"$.ListPhoneNumbersOptedOut.phoneNumbers[*]",
350+
"$.ListSMSSandboxPhoneNumbers.PhoneNumbers[*].PhoneNumber",
351+
352+
// S3 (RedactionRulesExtractor.java for s3-2006-03-01)
353+
// ./services/s3/src/main/resources/codegen-resources/service-2.json
354+
"$.CompleteMultipartUpload.SSEKMSKeyId",
355+
"$.CreateSession.Credentials.SecretAccessKey",
356+
"$.CreateSession.Credentials.SessionToken",
357+
"$.GetBucketEncryption.ServerSideEncryptionConfiguration.Rules[*].ApplyServerSideEncryptionByDefault.KMSMasterKeyID",
358+
"$.GetBucketInventoryConfiguration.InventoryConfiguration.Destination.S3BucketDestination.Encryption.SSEKMS.KeyId",
359+
"$.GetObject.SSEKMSKeyId",
360+
"$.HeadObject.SSEKMSKeyId",
361+
"$.ListBucketInventoryConfigurations.InventoryConfigurationList[*].Destination.S3BucketDestination.Encryption.SSEKMS.KeyId",
362+
"$.UploadPart.SSEKMSKeyId",
363+
"$.UploadPartCopy.SSEKMSKeyId",
364+
365+
// https://github.com/DataDog/dd-trace-js/blob/5658024a4a0c78476be6b04c13e821b6f9f86762/packages/dd-trace/src/payload-tagging/config/aws.json
299366
"$.Endpoints.*.Token",
300367
"$.PlatformApplication.*.PlatformCredential",
301368
"$.PlatformApplication.*.PlatformPrincipal",
302-
"$.Subscriptions.*.Endpoint",
303-
// Sns (Generated by RedactionRulesExtractor.java for sns-2010-03-31)
304-
"$.PhoneNumbers[*].PhoneNumber",
305-
"$.phoneNumbers[*]",
306-
// S3 (RedactionRulesExtractor.java for s3-2006-03-01)
307-
"$.Credentials.SecretAccessKey",
308-
"$.Credentials.SessionToken",
309-
"$.InventoryConfigurationList[*].Destination.S3BucketDestination.Encryption.SSEKMS.KeyId");
369+
"$.Subscriptions.*.Endpoint");
310370

311371
private ConfigDefaults() {}
312372
}

‎dd-trace-core/src/test/java/datadog/trace/payloadtags/RedactionRulesExtractor.java‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,9 @@ public static void main(String[] args) throws IOException {
6464
if (inputObject != null) {
6565
String inputShape = (String) inputObject.get("shape");
6666
collectSensitivePaths(
67-
operationName,
6867
(Map<String, Object>) allShapes.get(inputShape),
6968
allShapes,
70-
"$",
69+
operationName,
7170
requestSensitivePaths);
7271
}
7372

@@ -76,10 +75,9 @@ public static void main(String[] args) throws IOException {
7675
if (outputObject != null) {
7776
String outputShape = (String) outputObject.get("shape");
7877
collectSensitivePaths(
79-
operationName,
8078
(Map<String, Object>) allShapes.get(outputShape),
8179
allShapes,
82-
"$",
80+
operationName,
8381
responseSensitivePaths);
8482
}
8583

@@ -88,12 +86,9 @@ public static void main(String[] args) throws IOException {
8886
operationObject.getOrDefault("errors", Collections.emptyList());
8987
for (Map<String, Object> error : errors) {
9088
String errorShape = (String) error.get("shape");
91-
collectSensitivePaths(
92-
operationName,
93-
(Map<String, Object>) allShapes.get(errorShape),
94-
allShapes,
95-
"$",
96-
errorsSensitivePaths);
89+
Map<String, Object> shape = (Map<String, Object>) allShapes.get(errorShape);
90+
Set<String> collectedSensitivePath = errorsSensitivePaths;
91+
collectSensitivePaths(shape, allShapes, operationName, collectedSensitivePath);
9792
}
9893
}
9994

@@ -103,23 +98,36 @@ public static void main(String[] args) throws IOException {
10398
requestSensitivePaths.removeAll(commonSensitivePaths);
10499
responseSensitivePaths.removeAll(commonSensitivePaths);
105100

106-
System.out.println("\nCommon sensitive paths:\n" + String.join("\n", commonSensitivePaths));
107-
System.out.println("\nRequest sensitive paths:\n" + String.join("\n", requestSensitivePaths));
108-
System.out.println("\nResponse sensitive paths:\n" + String.join("\n", responseSensitivePaths));
109-
System.out.println("\nErrors sensitive paths:\n" + String.join("\n", errorsSensitivePaths));
101+
System.out.println(
102+
"\nCommon sensitive paths:\n\"" + String.join("\",\n\"", commonSensitivePaths) + "\"");
103+
System.out.println(
104+
"\nRequest sensitive paths:\n\"" + String.join("\",\n\"", requestSensitivePaths) + "\"");
105+
System.out.println(
106+
"\nResponse sensitive paths:\n\"" + String.join("\",\n\"", responseSensitivePaths) + "\"");
107+
System.out.println(
108+
"\nErrors sensitive paths:\n\"" + String.join("\",\n\"", errorsSensitivePaths) + "\"");
110109
Map<String, Object> metadata = (Map<String, Object>) map.get("metadata");
111110
System.out.println("serviceId: " + metadata.get("serviceId"));
112111
System.out.println("uid: " + metadata.get("uid"));
113112
}
114113

115114
private static void collectSensitivePaths(
115+
Map<String, Object> shape,
116+
Map<String, Object> allShapes,
116117
String operationName,
118+
Set<String> collectedSensitivePath) {
119+
collectSensitivePathsRecursively(
120+
shape, allShapes, "$." + operationName, "", collectedSensitivePath);
121+
}
122+
123+
private static void collectSensitivePathsRecursively(
117124
Map<String, Object> shape,
118125
Map<String, Object> allShapes,
126+
String prefix,
119127
String path,
120128
Set<String> sensitivePathsOut) {
121129
if ((boolean) shape.getOrDefault("sensitive", false)) {
122-
sensitivePathsOut.add(path);
130+
sensitivePathsOut.add(prefix + path);
123131
return;
124132
}
125133

@@ -132,20 +140,20 @@ private static void collectSensitivePaths(
132140
String memberName = member.getKey();
133141
Map<String, Object> memberObject = (Map<String, Object>) member.getValue();
134142
String memberShape = (String) memberObject.get("shape");
135-
collectSensitivePaths(
136-
operationName,
143+
collectSensitivePathsRecursively(
137144
(Map<String, Object>) allShapes.get(memberShape),
138145
allShapes,
146+
prefix,
139147
path + "." + memberName,
140148
sensitivePathsOut);
141149
}
142150
} else if ("list".equals(shapeType)) {
143151
Map<String, Object> member = (Map<String, Object>) shape.get("member");
144152
String memberShape = (String) member.get("shape");
145-
collectSensitivePaths(
146-
operationName,
153+
collectSensitivePathsRecursively(
147154
(Map<String, Object>) allShapes.get(memberShape),
148155
allShapes,
156+
prefix,
149157
path + "[*]",
150158
sensitivePathsOut);
151159
}

0 commit comments

Comments
 (0)