TST: Add CTIP Azure test for invalid FQDN filtering

Author: monoidic

intelmq/tests/bots/parsers/microsoft/ctip_azure.txt

@@ -1,4 +1,5 @@
 {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348339284870000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:45:28.4870","Malware":"Avalanche","ThreatCode":"B67-SS-TINBA","ThreatConfidence":"Low","TotalEncounters":3,"TLP":"Amber","SourceIp":"224.0.5.8","SourcePort":65116,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS 1","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"tinba","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTU5MDM2MDMyOC40ODc0MiwiaXAiOiIxMjcuMC4wLjEiLCJwb3J0Ijo2NTExNiwic2VydmVySXAiOiIxOTguMTguMTg1LjE2MiIsInNlcnZlclBvcnQiOjgwLCJkb21haW4iOiJleGFtcGxlLmNvbSIsImZhbWlseSI6InRpbmJhIiwibWFsd2FyZSI6e30sInJlc3BvbnNlIjoiUmVzcG9uc2UiLCJoYW5kbGVyIjoidGluYmEiLCJ0eXBlIjoiSHR0cCJ9"}
 {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348340630510000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:47:43.0510","Malware":"Avalanche","ThreatCode":"B67-SS-MATSNU","ThreatConfidence":"High","TotalEncounters":5,"TLP":"YELLOW","SourceIp":"224.0.5.8","SourcePort":49296,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64497","SourceIpAsnOrgName":"Example AS 2","SourceIpCountryCode":"AT","SourceIpRegion":"Vienna","SourceIpCity":"Vienna","SourceIpPostalCode":"1060","SourceIpLatitude":48.1951,"SourceIpLongitude":16.3483,"SourceIpMetroCode":0,"SourceIpAreaCode":9,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"matsnu5","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"dGhpcyBpcyBqdXN0IHNvbWUgdGV4dA=="}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"}
-{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiMTAuMC4wLjEiLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}

intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py

@@ -156,6 +156,39 @@
         'tlp': 'GREEN',
         'extra.source.connection_type': 'Cellular',
     },
+    {
+        '__type': 'Event',
+        'classification.type': 'infected-system',
+        'destination.ip': '10.0.0.1',
+        'destination.port': 80,
+        'event_description.text': 'Microsoft.DCU.CTIP.Sinkhole',
+        'extra.custom_field1': 'andromeda210',
+        'extra.malware': 'Avalanche',
+        'extra.payload.family': 'andromeda',
+        'extra.payload.handler': 'handler1',
+        'extra.payload.ip': '224.0.5.8',
+        'extra.payload.port': 28285,
+        'extra.payload.response': 'HttpOk',
+        'extra.payload.server.ip': '10.0.0.1',
+        'extra.payload.server.port': 80,
+        'extra.payload.timestamp': '2021-05-10T15:47:42.242316+00:00',
+        'extra.total_encounters': 2,
+        'feed.accuracy': 20.0,
+        'feed.name': 'Microsoft.DCU.CTIP.Infected',
+        'malware.name': 'b67-ss-gamarue',
+        'protocol.application': 'http',
+        'raw': 'eyJEYXRhRmVlZCI6Ik1pY3Jvc29mdC5EQ1UuQ1RJUC5JbmZlY3RlZCIsIlNvdXJjZWRGcm9tIjoiTWljcm9zb2Z0LkRDVS5DVElQLlNpbmtob2xlIiwiRGF0ZVRpbWVSZWNlaXZlZFV0YyI6MTMyNjUxMzUyNjIyNDIwMDAwLCJEYXRlVGltZVJlY2VpdmVkVXRjVHh0IjoiTW9uZGF5IE1heSAxMCAyMDIxIDE1OjQ3OjQyLjI0MjAiLCJNYWx3YXJlIjoiQXZhbGFuY2hlIiwiVGhyZWF0Q29kZSI6IkI2Ny1TUy1HYW1hcnVlIiwiVGhyZWF0Q29uZmlkZW5jZSI6IkxvdyIsIlRvdGFsRW5jb3VudGVycyI6MiwiVExQIjoiR3JlZW4iLCJTb3VyY2VJcCI6IjIyNC4wLjUuOCIsIlNvdXJjZVBvcnQiOjI4Mjg1LCJEZXN0aW5hdGlvbklwIjoiMTAuMC4wLjEiLCJEZXN0aW5hdGlvblBvcnQiOjgwLCJTb3VyY2VJcEluZm8iOnsiU291cmNlSXBBc25OdW1iZXIiOiI2NDQ5NiIsIlNvdXJjZUlwQXNuT3JnTmFtZSI6IkV4YW1wbGUgQVMiLCJTb3VyY2VJcENvdW50cnlDb2RlIjoiQVQiLCJTb3VyY2VJcFJlZ2lvbiI6IiIsIlNvdXJjZUlwQ2l0eSI6IiIsIlNvdXJjZUlwUG9zdGFsQ29kZSI6IiIsIlNvdXJjZUlwTGF0aXR1ZGUiOjQ4LjIsIlNvdXJjZUlwTG9uZ2l0dWRlIjoxNi4zNjY3LCJTb3VyY2VJcE1ldHJvQ29kZSI6MCwiU291cmNlSXBBcmVhQ29kZSI6MCwiU291cmNlSXBDb25uZWN0aW9uVHlwZSI6IkNlbGx1bGFyIiwiU291cmNlSXB2NEludCI6Mzc1ODA5NzY3Mn0sIkh0dHBJbmZvIjp7Ikh0dHBIb3N0IjoiIiwiSHR0cFJlcXVlc3QiOiIiLCJIdHRwTWV0aG9kIjoiIiwiSHR0cFJlZmVycmVyIjoiIiwiSHR0cFVzZXJBZ2VudCI6IiIsIkh0dHBWZXJzaW9uIjoiIn0sIkN1c3RvbUluZm8iOnsiQ3VzdG9tRmllbGQxIjoiYW5kcm9tZWRhMjEwIiwiQ3VzdG9tRmllbGQyIjoiIiwiQ3VzdG9tRmllbGQzIjoiIiwiQ3VzdG9tRmllbGQ0IjoiIiwiQ3VzdG9tRmllbGQ1IjoiIn0sIlBheWxvYWQiOiJleUowY3lJNk1UWXlNRFkyTVRZMk1pNHlOREl6TVRZc0ltbHdJam9pTWpJMExqQXVOUzQ0SWl3aWNHOXlkQ0k2TWpneU9EVXNJbk5sY25abGNrbHdJam9pTVRBdU1DNHdMakVpTENKelpYSjJaWEpRYjNKMElqbzRNQ3dpWkc5dFlXbHVJam9pTVRBdU1DNHdMakVpTENKbVlXMXBiSGtpT2lKaGJtUnliMjFsWkdFaUxDSnRZV3gzWVhKbElqcDdmU3dpY21WemNHOXVjMlVpT2lKSWRIUndUMnNpTENKb1lXNWtiR1Z5SWpvaWFHRnVaR3hsY2pFaUxDSjBlWEJsSWpvaVNIUjBjQ0o5In0=',
+        'source.as_name': 'Example AS',
+        'source.asn': 64496,
+        'source.geolocation.cc': 'AT',
+        'source.geolocation.latitude': 48.2,
+        'source.geolocation.longitude': 16.3667,
+        'source.ip': '224.0.5.8',
+        'source.port': 28285,
+        'time.source': '2021-05-10T15:47:42.241999+00:00',
+        'tlp': 'GREEN',
+        'extra.source.connection_type': 'Cellular',
+    },
 ]