Fortinet FortiGate WebSession Parsers Parsing Fix & Additions

Parsers/ASimWebSession/Parsers/ASimWebSessionFortinetFortiGate.yaml

@@ -1,7 +1,7 @@
 Parser:
   Title: Web Session ASIM parser for Fortinet FortiGate
-  Version: '0.1'
-  LastUpdated: Nov 11th, 2023
+  Version: '0.1.1'
+  LastUpdated: Nov 4, 2024
 Product:
   Name: Fortinet FortiGate
 Normalization:
@@ -11,7 +11,7 @@ References:
 - Title: ASIM Web Session Schema
   Link: https://aka.ms/ASimWebSessionDoc
 - Title: ASIM
-  Link: https:/aka.ms/AboutASIM
+  Link: https://aka.ms/AboutASIM
 - Title: web log fields
   Link: https://docs.fortinet.com/document/fortigate/7.4.0/fortios-log-message-reference/400992  
 - Title: Fortinet FortiGate CEF setup
@@ -52,10 +52,9 @@ ParserQuery: |
     | where DeviceVendor  == "Fortinet" 
         and DeviceProduct startswith "Fortigate"
         and Activity has_all ('webfilter', 'utm')
-    | extend 
-        EventResultDetails = "NA"
+    //| extend EventResultDetails = "NA" // HTTP response codes are not included in Fortigate logs.
     | lookup EventLookup on DeviceAction 
-    | project Activity,AdditionalExtensions,DestinationIP,DestinationPort,DeviceAction,DeviceInboundInterface,DeviceOutboundInterface,DeviceProduct,DeviceVersion,LogSeverity,Protocol,ReceivedBytes,SentBytes,SourceIP,SourcePort,TimeGenerated, DeviceExternalID, Type, _ItemId, Computer, EventResult, EventResultDetails, DvcAction, RequestURL, RequestContext, DestinationHostName, SourceHostName, SourceUserName, DestinationUserName
+    | project Activity,AdditionalExtensions,DestinationIP,DestinationPort,DeviceAction,DeviceInboundInterface,DeviceOutboundInterface,DeviceProduct,DeviceVersion,LogSeverity,Protocol,ReceivedBytes,SentBytes,SourceIP,SourcePort,TimeGenerated, DeviceExternalID, Type, _ItemId, Computer, EventResult, DvcAction, RequestURL, RequestContext, DestinationHostName, SourceHostName, SourceUserName, DestinationUserName, ApplicationProtocol
     | project-rename 
       Url                     = RequestURL
       , UrlCategory           = RequestContext
@@ -72,9 +71,9 @@ ParserQuery: |
       , SrcInterfaceName      = DeviceInboundInterface
       , SrcIpAddr             = SourceIP
       , SrcPortNumber         = SourcePort
+      , NetworkApplicationProtocol = ApplicationProtocol
       , DvcId                 = DeviceExternalID
       , EventUid              = _ItemId
-      , DstHostname           = DestinationHostName
       , SrcHostname           = SourceHostName
       , SrcUsername           = SourceUserName
       , DstUsername           = DestinationUserName
@@ -100,6 +99,7 @@ ParserQuery: |
             ['ad.agent']:string
         ) with (pair_delimiter=';', kv_delimiter='=')
     | parse AdditionalExtensions with * "x-forwarded-for=" HttpRequestXff:string ";" *
+    | invoke _ASIM_ResolveDstFQDN('DestinationHostName')
     | project-rename
         HttpReferrer            = ['ad.referralurl'],
         HttpRequestMethod       = ['ad.httpmethod'],
@@ -115,17 +115,21 @@ ParserQuery: |
         ThreatOriginalRiskLevel = FortinetFortiGatecrscore,
         SrcPackets              = FortinetFortiGatesentpkt,
         DstPackets              = FortinetFortiGatercvdpkt
-    | parse AdditionalExtensions with * "Method=" temp_HttpRequestMethod "|User-Agent=" temp_HttpUserAgent ";" *
+    | extend
+        temp_HttpRequestMethod = extract(@"rawdata=.*?Method=(.*?)(?:\||\;|$)", 1, AdditionalExtensions),
+        temp_HttpUserAgent = extract(@"rawdata=.*?User-Agent=(.*?)(?:\||\;|$)", 1, AdditionalExtensions)
     | extend 
         HttpRequestMethod = coalesce(temp_HttpRequestMethod,HttpRequestMethod),
         HttpUserAgent = coalesce(temp_HttpUserAgent,HttpUserAgent)
+        //HttpStatusCode = EventResultDetails // HTTP response codes are not included in Fortigate logs.
     | project-away temp_*
     | extend 
       EventCount               = int(1)
       , EventSchema            = "WebSession"
       , EventSchemaVersion     = "0.2.6"
       , EventType              = "HTTPsession"
       , EventVendor            = "Fortinet"
+      , EventProduct           = "Fortigate"
       , DvcIdType              = "Other"
       , NetworkBytes           = DstBytes + SrcBytes
       , EventEndTime           = TimeGenerated
@@ -148,4 +152,4 @@ ParserQuery: |
         Rule      = tostring(RuleNumber)
     | project-away Protocol, AdditionalExtensions, NetworkProtocolNumber
   };
-  parser (disabled=disabled)
+  parser (disabled=disabled)

Parsers/ASimWebSession/Parsers/vimWebSessionFortinetFortiGate.yaml

@@ -1,7 +1,7 @@
 Parser:
   Title: Web Session ASIM filtering parser for Fortinet FortiGate
-  Version: '0.1'
-  LastUpdated: Nov 11th, 2023
+  Version: '0.1.1'
+  LastUpdated: Nov 4, 2024
 Product:
   Name: Fortinet FortiGate
 Normalization:
@@ -11,7 +11,7 @@ References:
   - Title: ASIM Web Session Schema
     Link: https://aka.ms/ASimWebSessionDoc
   - Title: ASIM
-    Link: https:/aka.ms/AboutASIM
+    Link: https://aka.ms/AboutASIM
   - Title: web log fields
     Link: https://docs.fortinet.com/document/fortigate/7.4.0/fortios-log-message-reference/400992
   - Title: Fortinet FortiGate CEF setup
@@ -43,9 +43,6 @@ ParserParams:
   - Name: httpuseragent_has_any
     Type: dynamic 
     Default: dynamic([])
-  - Name: eventresultdetails_in
-    Type: dynamic 
-    Default: dynamic([])
   - Name: eventresult
     Type: string
     Default: '*'
@@ -60,7 +57,6 @@ ParserQuery: |
     ipaddr_has_any_prefix:dynamic    = dynamic([]), 
     url_has_any:dynamic              = dynamic([]),
     httpuseragent_has_any:dynamic    = dynamic([]),
-    eventresultdetails_in:dynamic    = dynamic([]),
     eventresult:string               = '*',
     disabled:bool                    = false
   ){
@@ -110,12 +106,9 @@ ParserQuery: |
         "No match") 
     | where ASimMatchingIpAddr != "No match" 
     | project-away temp_*
-    | extend 
-        EventResultDetails = "NA"
-    | where (array_length(eventresultdetails_in) == 0 or tostring(EventResultDetails) has_any(eventresultdetails_in)) 
     | lookup EventLookup on DeviceAction 
     | where (eventresult == '*' or EventResult =~ eventresult)
-    | project Activity,AdditionalExtensions,DestinationIP,DestinationPort,DeviceAction,DeviceInboundInterface,DeviceOutboundInterface,DeviceProduct,DeviceVersion,LogSeverity,Protocol,ReceivedBytes,SentBytes,SourceIP,SourcePort,TimeGenerated, DeviceExternalID, Type, _ItemId, Computer, EventResult, EventResultDetails, DvcAction, RequestURL, RequestContext, DestinationHostName, SourceHostName, SourceUserName, DestinationUserName, ASimMatchingIpAddr
+    | project Activity,AdditionalExtensions,DestinationIP,DestinationPort,DeviceAction,DeviceInboundInterface,DeviceOutboundInterface,DeviceProduct,DeviceVersion,LogSeverity,Protocol,ReceivedBytes,SentBytes,SourceIP,SourcePort,TimeGenerated, DeviceExternalID, Type, _ItemId, Computer, EventResult, DvcAction, RequestURL, RequestContext, DestinationHostName, SourceHostName, SourceUserName, DestinationUserName, ASimMatchingIpAddr, ApplicationProtocol
     | project-rename 
       Url                     = RequestURL
       , UrlCategory           = RequestContext
@@ -132,9 +125,9 @@ ParserQuery: |
       , SrcInterfaceName      = DeviceInboundInterface
       , SrcIpAddr             = SourceIP
       , SrcPortNumber         = SourcePort
+      , NetworkApplicationProtocol = ApplicationProtocol
       , DvcId                 = DeviceExternalID
       , EventUid              = _ItemId
-      , DstHostname           = DestinationHostName
       , SrcHostname           = SourceHostName
       , SrcUsername           = SourceUserName
       , DstUsername           = DestinationUserName
@@ -160,6 +153,7 @@ ParserQuery: |
             ['ad.agent']:string
         ) with (pair_delimiter=';', kv_delimiter='=')
     | parse AdditionalExtensions with * "x-forwarded-for=" HttpRequestXff:string ";" *
+    | invoke _ASIM_ResolveDstFQDN('DestinationHostName')
     | project-rename
         HttpReferrer            = ['ad.referralurl'],
         HttpRequestMethod       = ['ad.httpmethod'],
@@ -175,7 +169,9 @@ ParserQuery: |
         ThreatOriginalRiskLevel = FortinetFortiGatecrscore,
         SrcPackets              = FortinetFortiGatesentpkt,
         DstPackets              = FortinetFortiGatercvdpkt
-    | parse AdditionalExtensions with * "Method=" temp_HttpRequestMethod "|User-Agent=" temp_HttpUserAgent ";" *
+    | extend 
+        temp_HttpRequestMethod = extract(@"rawdata=.*?Method=(.*?)(?:\||\;|$)", 1, AdditionalExtensions),
+        temp_HttpUserAgent = extract(@"rawdata=.*?User-Agent=(.*?)(?:\||\;|$)", 1, AdditionalExtensions)
     | extend 
         HttpRequestMethod = coalesce(temp_HttpRequestMethod,HttpRequestMethod),
         HttpUserAgent = coalesce(temp_HttpUserAgent,HttpUserAgent)
@@ -187,6 +183,7 @@ ParserQuery: |
         , EventSchemaVersion     = "0.2.6"
         , EventType              = "HTTPsession"
         , EventVendor            = "Fortinet"
+        , EventProduct           = "Fortigate"
         , DvcIdType              = "Other"
         , NetworkBytes           = DstBytes + SrcBytes
         , EventEndTime           = TimeGenerated
@@ -209,4 +206,4 @@ ParserQuery: |
         Rule      = tostring(RuleNumber)
     | project-away Protocol, AdditionalExtensions, NetworkProtocolNumber
   };
-  parser (starttime=starttime, endtime=endtime, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, ipaddr_has_any_prefix=ipaddr_has_any_prefix, url_has_any=url_has_any, httpuseragent_has_any=httpuseragent_has_any, eventresultdetails_in=eventresultdetails_in, eventresult=eventresult, disabled=disabled)
+  parser (starttime=starttime, endtime=endtime, srcipaddr_has_any_prefix=srcipaddr_has_any_prefix, ipaddr_has_any_prefix=ipaddr_has_any_prefix, url_has_any=url_has_any, httpuseragent_has_any=httpuseragent_has_any, eventresult=eventresult, disabled=disabled)

Parsers/ASimWebSession/Tests/Fortinet_FortiGate_ASimWebSession_DataTest.csv

@@ -0,0 +1,23 @@
+Result
+"(0) Error: 30 invalid value(s) (up to 10 listed) in 29949 records (99.83%) for field [DstHostname] of type [Hostname]: [""mask.icloud.com"",""mask-h2.icloud.com"",""mask-api.icloud.com""] (Schema:WebSession)"
+"(0) Error: 30 invalid value(s) (up to 10 listed) in 29949 records (99.83%) for field [Hostname] of type [Hostname]: [""mask.icloud.com"",""mask-h2.icloud.com"",""mask-api.icloud.com""] (Schema:WebSession)"
+"(2) Info: Empty value in 1 records (0.0%)  in optional field [UrlCategory] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [DstGeoCountry] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [DstPackets] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [DstUsername] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [DstZone] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [HttpReferrer] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [HttpRequestMethod] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [HttpRequestXff] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [HttpUserAgent] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [NetworkDuration] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [NetworkPackets] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [NetworkSessionId] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [RuleNumber] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [Rule] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [SrcGeoCountry] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [SrcPackets] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [SrcUsername] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [SrcZone] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in optional field [ThreatOriginalRiskLevel] (Schema:WebSession)"
+"(2) Info: Empty value in 30000 records (100.0%)  in recommended field [SrcHostname] (Schema:WebSession)"

Parsers/ASimWebSession/Tests/Fortinet_FortiGate_ASimWebSession_SchemaTest.csv

@@ -0,0 +1,103 @@
+Result
+"(0) Error: Missing recommended alias [HttpStatusCode] aliasing existing column [EventResultDetails]"
+"(1) Warning: Missing recommended field [ASimMatchingIpAddr]"
+"(1) Warning: Missing recommended field [DvcDomain]"
+"(1) Warning: Missing recommended field [DvcIpAddr]"
+"(1) Warning: Missing recommended field [SrcDomain]"
+"(2) Info: Missing optional alias [Hash] aliasing non-existent column [MD5|SHA1|SHA256|SHA512]"
+"(2) Info: Missing optional alias [InnerVlanId] aliasing non-existent column [SrcVlanId]"
+"(2) Info: Missing optional alias [OuterVlanId] aliasing non-existent column [DstVlanId]"
+"(2) Info: Missing optional field [AdditionalFields]"
+"(2) Info: Missing optional field [DstAppId]"
+"(2) Info: Missing optional field [DstAppName]"
+"(2) Info: Missing optional field [DstAppType]"
+"(2) Info: Missing optional field [DstDescription]"
+"(2) Info: Missing optional field [DstDeviceType]"
+"(2) Info: Missing optional field [DstDomain]"
+"(2) Info: Missing optional field [DstDvcId]"
+"(2) Info: Missing optional field [DstFQDN]"
+"(2) Info: Missing optional field [DstGeoCity]"
+"(2) Info: Missing optional field [DstGeoLatitude]"
+"(2) Info: Missing optional field [DstGeoLongitude]"
+"(2) Info: Missing optional field [DstGeoRegion]"
+"(2) Info: Missing optional field [DstInterfaceGuid]"
+"(2) Info: Missing optional field [DstMacAddr]"
+"(2) Info: Missing optional field [DstNatIpAddr]"
+"(2) Info: Missing optional field [DstNatPortNumber]"
+"(2) Info: Missing optional field [DstOriginalUserType]"
+"(2) Info: Missing optional field [DstUserId]"
+"(2) Info: Missing optional field [DstUserType]"
+"(2) Info: Missing optional field [DstVlanId]"
+"(2) Info: Missing optional field [DvcDescription]"
+"(2) Info: Missing optional field [DvcFQDN]"
+"(2) Info: Missing optional field [DvcInboundInterface]"
+"(2) Info: Missing optional field [DvcMacAddr]"
+"(2) Info: Missing optional field [DvcOutboundInterface]"
+"(2) Info: Missing optional field [DvcScopeId]"
+"(2) Info: Missing optional field [DvcScope]"
+"(2) Info: Missing optional field [DvcZone]"
+"(2) Info: Missing optional field [EventOriginalResultDetails]"
+"(2) Info: Missing optional field [EventOriginalSubType]"
+"(2) Info: Missing optional field [EventOriginalType]"
+"(2) Info: Missing optional field [EventOriginalUid]"
+"(2) Info: Missing optional field [EventOwner]"
+"(2) Info: Missing optional field [EventReportUrl]"
+"(2) Info: Missing optional field [EventSubType]"
+"(2) Info: Missing optional field [FileContentType]"
+"(2) Info: Missing optional field [FileMD5]"
+"(2) Info: Missing optional field [FileName]"
+"(2) Info: Missing optional field [FileSHA1]"
+"(2) Info: Missing optional field [FileSHA256]"
+"(2) Info: Missing optional field [FileSHA512]"
+"(2) Info: Missing optional field [FileSize]"
+"(2) Info: Missing optional field [HttpContentFormat]"
+"(2) Info: Missing optional field [HttpContentType]"
+"(2) Info: Missing optional field [HttpCookie]"
+"(2) Info: Missing optional field [HttpHost]"
+"(2) Info: Missing optional field [HttpIsProxied]"
+"(2) Info: Missing optional field [HttpRequestBodyBytes]"
+"(2) Info: Missing optional field [HttpRequestCacheControl]"
+"(2) Info: Missing optional field [HttpRequestHeaderCount]"
+"(2) Info: Missing optional field [HttpRequestTime]"
+"(2) Info: Missing optional field [HttpResponseBodyBytes]"
+"(2) Info: Missing optional field [HttpResponseCacheControl]"
+"(2) Info: Missing optional field [HttpResponseExpires]"
+"(2) Info: Missing optional field [HttpResponseHeaderCount]"
+"(2) Info: Missing optional field [HttpResponseTime]"
+"(2) Info: Missing optional field [HttpVersion]"
+"(2) Info: Missing optional field [NetworkConnectionHistory]"
+"(2) Info: Missing optional field [NetworkDirection]"
+"(2) Info: Missing optional field [NetworkIcmpCode]"
+"(2) Info: Missing optional field [NetworkIcmpType]"
+"(2) Info: Missing optional field [RuleName]"
+"(2) Info: Missing optional field [SrcAppId]"
+"(2) Info: Missing optional field [SrcAppName]"
+"(2) Info: Missing optional field [SrcAppType]"
+"(2) Info: Missing optional field [SrcDescription]"
+"(2) Info: Missing optional field [SrcDeviceType]"
+"(2) Info: Missing optional field [SrcDvcId]"
+"(2) Info: Missing optional field [SrcFQDN]"
+"(2) Info: Missing optional field [SrcGeoCity]"
+"(2) Info: Missing optional field [SrcGeoLatitude]"
+"(2) Info: Missing optional field [SrcGeoLongitude]"
+"(2) Info: Missing optional field [SrcGeoRegion]"
+"(2) Info: Missing optional field [SrcInterfaceGuid]"
+"(2) Info: Missing optional field [SrcMacAddr]"
+"(2) Info: Missing optional field [SrcNatIpAddr]"
+"(2) Info: Missing optional field [SrcNatPortNumber]"
+"(2) Info: Missing optional field [SrcOriginalUserType]"
+"(2) Info: Missing optional field [SrcUserId]"
+"(2) Info: Missing optional field [SrcUserType]"
+"(2) Info: Missing optional field [SrcVlanId]"
+"(2) Info: Missing optional field [ThreatCategory]"
+"(2) Info: Missing optional field [ThreatConfidence]"
+"(2) Info: Missing optional field [ThreatField]"
+"(2) Info: Missing optional field [ThreatFirstReportedTime]"
+"(2) Info: Missing optional field [ThreatId]"
+"(2) Info: Missing optional field [ThreatIpAddr]"
+"(2) Info: Missing optional field [ThreatIsActive]"
+"(2) Info: Missing optional field [ThreatLastReportedTime]"
+"(2) Info: Missing optional field [ThreatName]"
+"(2) Info: Missing optional field [ThreatOriginalConfidence]"
+"(2) Info: Missing optional field [ThreatRiskLevel]"
+"(2) Info: Missing optional field [UrlOriginal]"