Skip to content

Commit 338b8c1

Browse files
aws-sdk-dotnet-automationaws-sdk-dotnet-automation
committed
Added CloudWatch Logs Transformer support for converting CloudTrail, VPC Flow, EKS Audit, AWS WAF and Route53 Resolver logs to OCSF v1.1 format.
1 parent a0e23a8 commit 338b8c1

19 files changed

+643
-59
lines changed

‎generator/ServiceModels/logs/logs-2014-03-28.api.json

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2456,6 +2456,16 @@
24562456
"min":1
24572457
},
24582458
"EventNumber":{"type":"long"},
2459+
"EventSource":{
2460+
"type":"string",
2461+
"enum":[
2462+
"CloudTrail",
2463+
"Route53Resolver",
2464+
"VPCFlow",
2465+
"EKSAudit",
2466+
"AWSWAF"
2467+
]
2468+
},
24592469
"EventsLimit":{
24602470
"type":"integer",
24612471
"max":10000,
@@ -3447,6 +3457,10 @@
34473457
"max":128,
34483458
"min":1
34493459
},
3460+
"OCSFVersion":{
3461+
"type":"string",
3462+
"enum":["V1.1"]
3463+
},
34503464
"OpenSearchApplication":{
34513465
"type":"structure",
34523466
"members":{
@@ -3661,6 +3675,18 @@
36613675
"source":{"shape":"Source"}
36623676
}
36633677
},
3678+
"ParseToOCSF":{
3679+
"type":"structure",
3680+
"required":[
3681+
"eventSource",
3682+
"ocsfVersion"
3683+
],
3684+
"members":{
3685+
"source":{"shape":"Source"},
3686+
"eventSource":{"shape":"EventSource"},
3687+
"ocsfVersion":{"shape":"OCSFVersion"}
3688+
}
3689+
},
36643690
"ParseVPC":{
36653691
"type":"structure",
36663692
"members":{
@@ -3751,6 +3777,7 @@
37513777
"parseJSON":{"shape":"ParseJSON"},
37523778
"parseKeyValue":{"shape":"ParseKeyValue"},
37533779
"parseRoute53":{"shape":"ParseRoute53"},
3780+
"parseToOCSF":{"shape":"ParseToOCSF"},
37543781
"parsePostgres":{"shape":"ParsePostgres"},
37553782
"parseVPC":{"shape":"ParseVPC"},
37563783
"parseWAF":{"shape":"ParseWAF"},

‎generator/ServiceModels/logs/logs-2014-03-28.docs.json

Lines changed: 21 additions & 2 deletions
Large diffs are not rendered by default.

‎generator/ServiceModels/logs/logs-2014-03-28.normal.json

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,7 @@
11431143
{"shape":"ServiceUnavailableException"},
11441144
{"shape":"UnrecognizedClientException"}
11451145
],
1146-
"documentation":"<p>Uploads a batch of log events to the specified log stream.</p> <important> <p>The sequence token is now ignored in <code>PutLogEvents</code> actions. <code>PutLogEvents</code> actions are always accepted and never return <code>InvalidSequenceTokenException</code> or <code>DataAlreadyAcceptedException</code> even if the sequence token is not valid. You can use parallel <code>PutLogEvents</code> actions on the same log stream. </p> </important> <p>The batch of events must satisfy the following constraints:</p> <ul> <li> <p>The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.</p> </li> <li> <p>None of the log events in the batch can be more than 2 hours in the future.</p> </li> <li> <p>None of the log events in the batch can be more than 14 days in the past. Also, none of the log events can be from earlier than the retention period of the log group.</p> </li> <li> <p>The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the event occurred, expressed as the number of milliseconds after <code>Jan 1, 1970 00:00:00 UTC</code>. (In Amazon Web Services Tools for PowerShell and the Amazon Web Services SDK for .NET, the timestamp is specified in .NET format: <code>yyyy-mm-ddThh:mm:ss</code>. For example, <code>2017-09-15T13:45:30</code>.) </p> </li> <li> <p>A batch of log events in a single request cannot span more than 24 hours. Otherwise, the operation fails.</p> </li> <li> <p>Each log event can be no larger than 1 MB.</p> </li> <li> <p>The maximum number of log events in a batch is 10,000.</p> </li> <li> <important> <p>The quota of five requests per second per log stream has been removed. Instead, <code>PutLogEvents</code> actions are throttled based on a per-second per-account quota. You can request an increase to the per-second throttling quota by using the Service Quotas service.</p> </important> </li> </ul> <p>If a call to <code>PutLogEvents</code> returns \"UnrecognizedClientException\" the most likely cause is a non-valid Amazon Web Services access key ID or secret key. </p>"
1146+
"documentation":"<p>Uploads a batch of log events to the specified log stream.</p> <important> <p>The sequence token is now ignored in <code>PutLogEvents</code> actions. <code>PutLogEvents</code> actions are always accepted and never return <code>InvalidSequenceTokenException</code> or <code>DataAlreadyAcceptedException</code> even if the sequence token is not valid. You can use parallel <code>PutLogEvents</code> actions on the same log stream. </p> </important> <p>The batch of events must satisfy the following constraints:</p> <ul> <li> <p>The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.</p> </li> <li> <p>Events more than 2 hours in the future are rejected while processing remaining valid events.</p> </li> <li> <p>Events older than 14 days or preceding the log group's retention period are rejected while processing remaining valid events.</p> </li> <li> <p>The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the event occurred, expressed as the number of milliseconds after <code>Jan 1, 1970 00:00:00 UTC</code>. (In Amazon Web Services Tools for PowerShell and the Amazon Web Services SDK for .NET, the timestamp is specified in .NET format: <code>yyyy-mm-ddThh:mm:ss</code>. For example, <code>2017-09-15T13:45:30</code>.) </p> </li> <li> <p> A batch of log events in a single request must be in a chronological order. Otherwise, the operation fails.</p> </li> <li> <p>Each log event can be no larger than 1 MB.</p> </li> <li> <p>The maximum number of log events in a batch is 10,000.</p> </li> <li> <p>For valid events (within 14 days in the past to 2 hours in future), the time span in a single batch cannot exceed 24 hours. Otherwise, the operation fails.</p> </li> </ul> <important> <p>The quota of five requests per second per log stream has been removed. Instead, <code>PutLogEvents</code> actions are throttled based on a per-second per-account quota. You can request an increase to the per-second throttling quota by using the Service Quotas service.</p> </important> <p>If a call to <code>PutLogEvents</code> returns \"UnrecognizedClientException\" the most likely cause is a non-valid Amazon Web Services access key ID or secret key. </p>"
11471147
},
11481148
"PutMetricFilter":{
11491149
"name":"PutMetricFilter",
@@ -3255,6 +3255,16 @@
32553255
"min":1
32563256
},
32573257
"EventNumber":{"type":"long"},
3258+
"EventSource":{
3259+
"type":"string",
3260+
"enum":[
3261+
"CloudTrail",
3262+
"Route53Resolver",
3263+
"VPCFlow",
3264+
"EKSAudit",
3265+
"AWSWAF"
3266+
]
3267+
},
32583268
"EventsLimit":{
32593269
"type":"integer",
32603270
"max":10000,
@@ -4859,6 +4869,10 @@
48594869
"max":128,
48604870
"min":1
48614871
},
4872+
"OCSFVersion":{
4873+
"type":"string",
4874+
"enum":["V1.1"]
4875+
},
48624876
"OpenSearchApplication":{
48634877
"type":"structure",
48644878
"members":{
@@ -5238,6 +5252,28 @@
52385252
},
52395253
"documentation":"<p>Use this processor to parse Route 53 vended logs, extract fields, and and convert them into a JSON format. This processor always processes the entire log event message. For more information about this processor including examples, see <a href=\"https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch-Logs-Transformation.html#CloudWatch-Logs-Transformation-parseRoute53\"> parseRoute53</a>.</p> <important> <p>If you use this processor, it must be the first processor in your transformer.</p> </important>"
52405254
},
5255+
"ParseToOCSF":{
5256+
"type":"structure",
5257+
"required":[
5258+
"eventSource",
5259+
"ocsfVersion"
5260+
],
5261+
"members":{
5262+
"source":{
5263+
"shape":"Source",
5264+
"documentation":"<p>The path to the field in the log event that you want to parse. If you omit this value, the whole log message is parsed.</p>"
5265+
},
5266+
"eventSource":{
5267+
"shape":"EventSource",
5268+
"documentation":"<p>Specify the service or process that produces the log events that will be converted with this processor.</p>"
5269+
},
5270+
"ocsfVersion":{
5271+
"shape":"OCSFVersion",
5272+
"documentation":"<p>Specify which version of the OCSF schema to use for the transformed log events.</p>"
5273+
}
5274+
},
5275+
"documentation":"<p>This processor converts logs into <a href=\"https://ocsf.io\">Open Cybersecurity Schema Framework (OCSF)</a> events.</p> <p>For more information about this processor including examples, see <a href=\"https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch-Logs-Transformation.html#CloudWatch-Logs-Transformation-parseToOCSF\"> parseToOSCF</a> in the <i>CloudWatch Logs User Guide</i>.</p>"
5276+
},
52415277
"ParseVPC":{
52425278
"type":"structure",
52435279
"members":{
@@ -5395,6 +5431,10 @@
53955431
"shape":"ParseRoute53",
53965432
"documentation":"<p>Use this parameter to include the <a href=\"https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch-Logs-Transformation.html#CloudWatch-Logs-Transformation-parseRoute53\"> parseRoute53</a> processor in your transformer.</p> <p>If you use this processor, it must be the first processor in your transformer.</p>"
53975433
},
5434+
"parseToOCSF":{
5435+
"shape":"ParseToOCSF",
5436+
"documentation":"<p>Use this processor to convert logs into Open Cybersecurity Schema Framework (OCSF) format</p>"
5437+
},
53985438
"parsePostgres":{
53995439
"shape":"ParsePostgres",
54005440
"documentation":"<p>Use this parameter to include the <a href=\"https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch-Logs-Transformation.html#CloudWatch-Logs-Transformation-parsePostGres\"> parsePostGres</a> processor in your transformer.</p> <p>If you use this processor, it must be the first processor in your transformer.</p>"
@@ -5591,7 +5631,7 @@
55915631
},
55925632
"logType":{
55935633
"shape":"LogType",
5594-
"documentation":"<p>Defines the type of log that the source is sending.</p> <ul> <li> <p>For Amazon Bedrock, the valid value is <code>APPLICATION_LOGS</code>.</p> </li> <li> <p>For CloudFront, the valid value is <code>ACCESS_LOGS</code>.</p> </li> <li> <p>For Amazon CodeWhisperer, the valid value is <code>EVENT_LOGS</code>.</p> </li> <li> <p>For Elemental MediaPackage, the valid values are <code>EGRESS_ACCESS_LOGS</code> and <code>INGRESS_ACCESS_LOGS</code>.</p> </li> <li> <p>For Elemental MediaTailor, the valid values are <code>AD_DECISION_SERVER_LOGS</code>, <code>MANIFEST_SERVICE_LOGS</code>, and <code>TRANSCODE_LOGS</code>.</p> </li> <li> <p>For IAM Identity Center, the valid value is <code>ERROR_LOGS</code>.</p> </li> <li> <p>For Amazon Q, the valid value is <code>EVENT_LOGS</code>.</p> </li> <li> <p>For Amazon SES mail manager, the valid value is <code>APPLICATION_LOG</code>.</p> </li> <li> <p>For Amazon WorkMail, the valid values are <code>ACCESS_CONTROL_LOGS</code>, <code>AUTHENTICATION_LOGS</code>, <code>WORKMAIL_AVAILABILITY_PROVIDER_LOGS</code>, <code>WORKMAIL_MAILBOX_ACCESS_LOGS</code>, and <code>WORKMAIL_PERSONAL_ACCESS_TOKEN_LOGS</code>.</p> </li> </ul>"
5634+
"documentation":"<p>Defines the type of log that the source is sending.</p> <ul> <li> <p>For Amazon Bedrock, the valid value is <code>APPLICATION_LOGS</code>.</p> </li> <li> <p>For CloudFront, the valid value is <code>ACCESS_LOGS</code>.</p> </li> <li> <p>For Amazon CodeWhisperer, the valid value is <code>EVENT_LOGS</code>.</p> </li> <li> <p>For Elemental MediaPackage, the valid values are <code>EGRESS_ACCESS_LOGS</code> and <code>INGRESS_ACCESS_LOGS</code>.</p> </li> <li> <p>For Elemental MediaTailor, the valid values are <code>AD_DECISION_SERVER_LOGS</code>, <code>MANIFEST_SERVICE_LOGS</code>, and <code>TRANSCODE_LOGS</code>.</p> </li> <li> <p>For Entity Resolution, the valid value is <code>WORKFLOW_LOGS</code>.</p> </li> <li> <p>For IAM Identity Center, the valid value is <code>ERROR_LOGS</code>.</p> </li> <li> <p>For Amazon Q, the valid value is <code>EVENT_LOGS</code>.</p> </li> <li> <p>For Amazon SES mail manager, the valid values are <code>APPLICATION_LOG</code> and <code>TRAFFIC_POLICY_DEBUG_LOGS</code>.</p> </li> <li> <p>For Amazon WorkMail, the valid values are <code>ACCESS_CONTROL_LOGS</code>, <code>AUTHENTICATION_LOGS</code>, <code>WORKMAIL_AVAILABILITY_PROVIDER_LOGS</code>, <code>WORKMAIL_MAILBOX_ACCESS_LOGS</code>, and <code>WORKMAIL_PERSONAL_ACCESS_TOKEN_LOGS</code>.</p> </li> </ul>"
55955635
},
55965636
"tags":{
55975637
"shape":"Tags",

‎sdk/code-analysis/ServiceAnalysis/CloudWatchLogs/Generated/PropertyValueRules.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1802,6 +1802,11 @@
18021802
<min>1</min>
18031803
<max>128</max>
18041804
</property-value-rule>
1805+
<property-value-rule>
1806+
<property>Amazon.CloudWatchLogs.Model.ParseToOCSF.Source</property>
1807+
<min>1</min>
1808+
<max>128</max>
1809+
</property-value-rule>
18051810
<property-value-rule>
18061811
<property>Amazon.CloudWatchLogs.Model.ParseVPC.Source</property>
18071812
<min>1</min>

‎sdk/src/Services/CloudWatchLogs/Generated/Model/Internal/MarshallTransformations/ParseToOCSFMarshaller.cs

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
/*
17+
* Do not modify this file. This file is generated from the logs-2014-03-28.normal.json service model.
18+
*/
19+
using System;
20+
using System.Collections.Generic;
21+
using System.Globalization;
22+
using System.IO;
23+
using System.Text;
24+
using System.Xml.Serialization;
25+
26+
using Amazon.CloudWatchLogs.Model;
27+
using Amazon.Runtime;
28+
using Amazon.Runtime.Internal;
29+
using Amazon.Runtime.Internal.Transform;
30+
using Amazon.Runtime.Internal.Util;
31+
using ThirdParty.Json.LitJson;
32+
33+
#pragma warning disable CS0612,CS0618
34+
namespace Amazon.CloudWatchLogs.Model.Internal.MarshallTransformations
35+
{
36+
/// <summary>
37+
/// ParseToOCSF Marshaller
38+
/// </summary>
39+
public class ParseToOCSFMarshaller : IRequestMarshaller<ParseToOCSF, JsonMarshallerContext>
40+
{
41+
/// <summary>
42+
/// Unmarshaller the response from the service to the response class.
43+
/// </summary>
44+
/// <param name="requestObject"></param>
45+
/// <param name="context"></param>
46+
/// <returns></returns>
47+
public void Marshall(ParseToOCSF requestObject, JsonMarshallerContext context)
48+
{
49+
if(requestObject == null)
50+
return;
51+
if(requestObject.IsSetEventSource())
52+
{
53+
context.Writer.WritePropertyName("eventSource");
54+
context.Writer.Write(requestObject.EventSource);
55+
}
56+
57+
if(requestObject.IsSetOcsfVersion())
58+
{
59+
context.Writer.WritePropertyName("ocsfVersion");
60+
context.Writer.Write(requestObject.OcsfVersion);
61+
}
62+
63+
if(requestObject.IsSetSource())
64+
{
65+
context.Writer.WritePropertyName("source");
66+
context.Writer.Write(requestObject.Source);
67+
}
68+
69+
}
70+
71+
/// <summary>
72+
/// Singleton Marshaller.
73+
/// </summary>
74+
public readonly static ParseToOCSFMarshaller Instance = new ParseToOCSFMarshaller();
75+
76+
}
77+
}

‎sdk/src/Services/CloudWatchLogs/Generated/Model/Internal/MarshallTransformations/ParseToOCSFUnmarshaller.cs

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
/*
17+
* Do not modify this file. This file is generated from the logs-2014-03-28.normal.json service model.
18+
*/
19+
using System;
20+
using System.Collections.Generic;
21+
using System.Globalization;
22+
using System.IO;
23+
using System.Net;
24+
using System.Text;
25+
using System.Xml.Serialization;
26+
27+
using Amazon.CloudWatchLogs.Model;
28+
using Amazon.Runtime;
29+
using Amazon.Runtime.Internal;
30+
using Amazon.Runtime.Internal.Transform;
31+
using Amazon.Runtime.Internal.Util;
32+
using ThirdParty.Json.LitJson;
33+
34+
#pragma warning disable CS0612,CS0618
35+
namespace Amazon.CloudWatchLogs.Model.Internal.MarshallTransformations
36+
{
37+
/// <summary>
38+
/// Response Unmarshaller for ParseToOCSF Object
39+
/// </summary>
40+
public class ParseToOCSFUnmarshaller : IUnmarshaller<ParseToOCSF, XmlUnmarshallerContext>, IUnmarshaller<ParseToOCSF, JsonUnmarshallerContext>
41+
{
42+
/// <summary>
43+
/// Unmarshaller the response from the service to the response class.
44+
/// </summary>
45+
/// <param name="context"></param>
46+
/// <returns></returns>
47+
ParseToOCSF IUnmarshaller<ParseToOCSF, XmlUnmarshallerContext>.Unmarshall(XmlUnmarshallerContext context)
48+
{
49+
throw new NotImplementedException();
50+
}
51+
52+
/// <summary>
53+
/// Unmarshaller the response from the service to the response class.
54+
/// </summary>
55+
/// <param name="context"></param>
56+
/// <returns>The unmarshalled object</returns>
57+
public ParseToOCSF Unmarshall(JsonUnmarshallerContext context)
58+
{
59+
ParseToOCSF unmarshalledObject = new ParseToOCSF();
60+
if (context.IsEmptyResponse)
61+
return null;
62+
context.Read();
63+
if (context.CurrentTokenType == JsonToken.Null)
64+
return null;
65+
66+
int targetDepth = context.CurrentDepth;
67+
while (context.ReadAtDepth(targetDepth))
68+
{
69+
if (context.TestExpression("eventSource", targetDepth))
70+
{
71+
var unmarshaller = StringUnmarshaller.Instance;
72+
unmarshalledObject.EventSource = unmarshaller.Unmarshall(context);
73+
continue;
74+
}
75+
if (context.TestExpression("ocsfVersion", targetDepth))
76+
{
77+
var unmarshaller = StringUnmarshaller.Instance;
78+
unmarshalledObject.OcsfVersion = unmarshaller.Unmarshall(context);
79+
continue;
80+
}
81+
if (context.TestExpression("source", targetDepth))
82+
{
83+
var unmarshaller = StringUnmarshaller.Instance;
84+
unmarshalledObject.Source = unmarshaller.Unmarshall(context);
85+
continue;
86+
}
87+
}
88+
return unmarshalledObject;
89+
}
90+
91+
92+
private static ParseToOCSFUnmarshaller _instance = new ParseToOCSFUnmarshaller();
93+
94+
/// <summary>
95+
/// Gets the singleton.
96+
/// </summary>
97+
public static ParseToOCSFUnmarshaller Instance
98+
{
99+
get
100+
{
101+
return _instance;
102+
}
103+
}
104+
}
105+
}

‎sdk/src/Services/CloudWatchLogs/Generated/Model/Internal/MarshallTransformations/ProcessorMarshaller.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,17 @@ public void Marshall(Processor requestObject, JsonMarshallerContext context)
202202
context.Writer.WriteObjectEnd();
203203
}
204204

205+
if(requestObject.IsSetParseToOCSF())
206+
{
207+
context.Writer.WritePropertyName("parseToOCSF");
208+
context.Writer.WriteObjectStart();
209+
210+
var marshaller = ParseToOCSFMarshaller.Instance;
211+
marshaller.Marshall(requestObject.ParseToOCSF, context);
212+
213+
context.Writer.WriteObjectEnd();
214+
}
215+
205216
if(requestObject.IsSetParseVPC())
206217
{
207218
context.Writer.WritePropertyName("parseVPC");

‎sdk/src/Services/CloudWatchLogs/Generated/Model/Internal/MarshallTransformations/ProcessorUnmarshaller.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,12 @@ public Processor Unmarshall(JsonUnmarshallerContext context)
150150
unmarshalledObject.ParseRoute53 = unmarshaller.Unmarshall(context);
151151
continue;
152152
}
153+
if (context.TestExpression("parseToOCSF", targetDepth))
154+
{
155+
var unmarshaller = ParseToOCSFUnmarshaller.Instance;
156+
unmarshalledObject.ParseToOCSF = unmarshaller.Unmarshall(context);
157+
continue;
158+
}
153159
if (context.TestExpression("parseVPC", targetDepth))
154160
{
155161
var unmarshaller = ParseVPCUnmarshaller.Instance;

0 commit comments

Comments
 (0)