Merge pull request #2144 from monoidic/ctip

MS CTIP Azure feed invalid hostname filtering

Author: sebix

CHANGELOG.md

@@ -52,7 +52,9 @@ CHANGELOG
   - SMTP Data
   - Telnet Login
   - VNC/RFB Login
-- `intelmq.bots.parsers.microsoft.parser_ctip`: New parameter `overwrite` (PR#2112 by Sebastian Wagner, fixes #2022).
+- `intelmq.bots.parsers.microsoft.parser_ctip`:
+  - New parameter `overwrite` (PR#2112 by Sebastian Wagner, fixes #2022).
+  - Fix handling of field `Payload.domain` if it contains the same IP address as `Payload.serverIp` (PR#2144 by Mikk Margus Möll and Sebastian Wagner).
 - `intelmq.bot.parsers.shodan.parser` (PR#2117 by Mikk Margus Möll):
   - Instead of keeping track of `extra.ftp.<something>.parameters`, FTP parameters are collected together into `extra.ftp.features` as a list of said features, reducing field count.
   - Shodan field `rsync.modules` is collected.

intelmq/bots/parsers/microsoft/parser_ctip.py

@@ -67,7 +67,7 @@
 
 import intelmq.lib.utils as utils
 from intelmq.lib.bot import ParserBot
-from intelmq.lib.harmonization import DateTime
+from intelmq.lib.harmonization import DateTime, FQDN
 
 INTERFLOW = {"additionalmetadata": "extra.additionalmetadata",
              "description": "event_description.text",
@@ -291,6 +291,10 @@ def parse_azure(self, line, report):
                 if payload_protocol:
                     # needs to overwrite a field previously parsed and written
                     event.add('protocol.application', payload_protocol, overwrite=True)  # "HTTP/1.1", save additionally
+            elif key == 'Payload.domain':
+                # Sometimes the destination address is also given as domain, ignore it here as we already save it as destination.ip (see https://github.com/certtools/intelmq/pull/2144)
+                if not FQDN.is_valid(value) and value == line.get('Payload.serverIp'):
+                    continue
             elif not value:
                 continue
             if AZURE[key] != '__IGNORE__':

intelmq/tests/bots/parsers/microsoft/ctip_azure.txt

@@ -1,4 +1,5 @@
 {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348339284870000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:45:28.4870","Malware":"Avalanche","ThreatCode":"B67-SS-TINBA","ThreatConfidence":"Low","TotalEncounters":3,"TLP":"Amber","SourceIp":"224.0.5.8","SourcePort":65116,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS 1","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"tinba","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTU5MDM2MDMyOC40ODc0MiwiaXAiOiIxMjcuMC4wLjEiLCJwb3J0Ijo2NTExNiwic2VydmVySXAiOiIxOTguMTguMTg1LjE2MiIsInNlcnZlclBvcnQiOjgwLCJkb21haW4iOiJleGFtcGxlLmNvbSIsImZhbWlseSI6InRpbmJhIiwibWFsd2FyZSI6e30sInJlc3BvbnNlIjoiUmVzcG9uc2UiLCJoYW5kbGVyIjoidGluYmEiLCJ0eXBlIjoiSHR0cCJ9"}
 {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348340630510000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:47:43.0510","Malware":"Avalanche","ThreatCode":"B67-SS-MATSNU","ThreatConfidence":"High","TotalEncounters":5,"TLP":"YELLOW","SourceIp":"224.0.5.8","SourcePort":49296,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64497","SourceIpAsnOrgName":"Example AS 2","SourceIpCountryCode":"AT","SourceIpRegion":"Vienna","SourceIpCity":"Vienna","SourceIpPostalCode":"1060","SourceIpLatitude":48.1951,"SourceIpLongitude":16.3483,"SourceIpMetroCode":0,"SourceIpAreaCode":9,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"matsnu5","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"dGhpcyBpcyBqdXN0IHNvbWUgdGV4dA=="}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"}
-{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiMTAuMC4wLjEiLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}

intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py

@@ -156,6 +156,39 @@
         'tlp': 'GREEN',
         'extra.source.connection_type': 'Cellular',
     },
+    {
+        '__type': 'Event',
+        'classification.type': 'infected-system',
+        'destination.ip': '10.0.0.1',
+        'destination.port': 80,
+        'event_description.text': 'Microsoft.DCU.CTIP.Sinkhole',
+        'extra.custom_field1': 'andromeda210',
+        'extra.malware': 'Avalanche',
+        'extra.payload.family': 'andromeda',
+        'extra.payload.handler': 'handler1',
+        'extra.payload.ip': '224.0.5.8',
+        'extra.payload.port': 28285,
+        'extra.payload.response': 'HttpOk',
+        'extra.payload.server.ip': '10.0.0.1',
+        'extra.payload.server.port': 80,
+        'extra.payload.timestamp': '2021-05-10T15:47:42.242316+00:00',
+        'extra.total_encounters': 2,
+        'feed.accuracy': 20.0,
+        'feed.name': 'Microsoft.DCU.CTIP.Infected',
+        'malware.name': 'b67-ss-gamarue',
+        'protocol.application': 'http',
+        'raw': 'eyJEYXRhRmVlZCI6Ik1pY3Jvc29mdC5EQ1UuQ1RJUC5JbmZlY3RlZCIsIlNvdXJjZWRGcm9tIjoiTWljcm9zb2Z0LkRDVS5DVElQLlNpbmtob2xlIiwiRGF0ZVRpbWVSZWNlaXZlZFV0YyI6MTMyNjUxMzUyNjIyNDIwMDAwLCJEYXRlVGltZVJlY2VpdmVkVXRjVHh0IjoiTW9uZGF5IE1heSAxMCAyMDIxIDE1OjQ3OjQyLjI0MjAiLCJNYWx3YXJlIjoiQXZhbGFuY2hlIiwiVGhyZWF0Q29kZSI6IkI2Ny1TUy1HYW1hcnVlIiwiVGhyZWF0Q29uZmlkZW5jZSI6IkxvdyIsIlRvdGFsRW5jb3VudGVycyI6MiwiVExQIjoiR3JlZW4iLCJTb3VyY2VJcCI6IjIyNC4wLjUuOCIsIlNvdXJjZVBvcnQiOjI4Mjg1LCJEZXN0aW5hdGlvbklwIjoiMTAuMC4wLjEiLCJEZXN0aW5hdGlvblBvcnQiOjgwLCJTb3VyY2VJcEluZm8iOnsiU291cmNlSXBBc25OdW1iZXIiOiI2NDQ5NiIsIlNvdXJjZUlwQXNuT3JnTmFtZSI6IkV4YW1wbGUgQVMiLCJTb3VyY2VJcENvdW50cnlDb2RlIjoiQVQiLCJTb3VyY2VJcFJlZ2lvbiI6IiIsIlNvdXJjZUlwQ2l0eSI6IiIsIlNvdXJjZUlwUG9zdGFsQ29kZSI6IiIsIlNvdXJjZUlwTGF0aXR1ZGUiOjQ4LjIsIlNvdXJjZUlwTG9uZ2l0dWRlIjoxNi4zNjY3LCJTb3VyY2VJcE1ldHJvQ29kZSI6MCwiU291cmNlSXBBcmVhQ29kZSI6MCwiU291cmNlSXBDb25uZWN0aW9uVHlwZSI6IkNlbGx1bGFyIiwiU291cmNlSXB2NEludCI6Mzc1ODA5NzY3Mn0sIkh0dHBJbmZvIjp7Ikh0dHBIb3N0IjoiIiwiSHR0cFJlcXVlc3QiOiIiLCJIdHRwTWV0aG9kIjoiIiwiSHR0cFJlZmVycmVyIjoiIiwiSHR0cFVzZXJBZ2VudCI6IiIsIkh0dHBWZXJzaW9uIjoiIn0sIkN1c3RvbUluZm8iOnsiQ3VzdG9tRmllbGQxIjoiYW5kcm9tZWRhMjEwIiwiQ3VzdG9tRmllbGQyIjoiIiwiQ3VzdG9tRmllbGQzIjoiIiwiQ3VzdG9tRmllbGQ0IjoiIiwiQ3VzdG9tRmllbGQ1IjoiIn0sIlBheWxvYWQiOiJleUowY3lJNk1UWXlNRFkyTVRZMk1pNHlOREl6TVRZc0ltbHdJam9pTWpJMExqQXVOUzQ0SWl3aWNHOXlkQ0k2TWpneU9EVXNJbk5sY25abGNrbHdJam9pTVRBdU1DNHdMakVpTENKelpYSjJaWEpRYjNKMElqbzRNQ3dpWkc5dFlXbHVJam9pTVRBdU1DNHdMakVpTENKbVlXMXBiSGtpT2lKaGJtUnliMjFsWkdFaUxDSnRZV3gzWVhKbElqcDdmU3dpY21WemNHOXVjMlVpT2lKSWRIUndUMnNpTENKb1lXNWtiR1Z5SWpvaWFHRnVaR3hsY2pFaUxDSjBlWEJsSWpvaVNIUjBjQ0o5In0=',
+        'source.as_name': 'Example AS',
+        'source.asn': 64496,
+        'source.geolocation.cc': 'AT',
+        'source.geolocation.latitude': 48.2,
+        'source.geolocation.longitude': 16.3667,
+        'source.ip': '224.0.5.8',
+        'source.port': 28285,
+        'time.source': '2021-05-10T15:47:42.241999+00:00',
+        'tlp': 'GREEN',
+        'extra.source.connection_type': 'Cellular',
+    },
 ]