Merge remote-tracking branch 'upstream/pr/1674' into develop

Author: Sebastian Wagner

intelmq/bots/parsers/generic/parser_csv.py

@@ -131,7 +131,7 @@ def parse_line(self, row, report):
                         value = self.type_translation[value]
                     elif not hasattr(self.parameters, 'type'):
                         continue
-                if event.add(key, value, raise_failure=False):
+                if event.add(key, value, raise_failure=False) is not False:
                     break
             else:
                 # if the value sill remains unadded we need to inform if the key is needed

intelmq/tests/bots/parsers/generic/invalid_sample_report.csv

@@ -1,2 +1,4 @@
 01/Aug/2018:09:49:41 +0200,127.0.0.1,-
-01/Aug/2018:09:49:41 +0200,127.0.0.1,example.com
+01/Aug/2018:09:49:41 +0200,127.0.0.1,example.com
+01/Aug/2018:09:49:41 +0200,-,example.com
+01/Aug/2018:09:49:41 +0200,127.0.0.1,.

intelmq/tests/bots/parsers/generic/test_parser_csv_invalid_values.py

@@ -21,14 +21,20 @@
                  "time.source": "2018-08-01T07:49:41+00:00",
                  "classification.type": "malware",
                  "source.ip": "127.0.0.1",
-                 "source.fqdn": "example.com",
-                 "raw": utils.base64_encode(SAMPLE_SPLIT[1]+'\r\n'),
+                 "raw": utils.base64_encode(SAMPLE_SPLIT[0]+'\r\n'),
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
 EXAMPLE_EVENT2 = EXAMPLE_EVENT.copy()
-del EXAMPLE_EVENT2['source.fqdn']
-EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[0]+'\r\n')
+EXAMPLE_EVENT2['source.fqdn'] = "example.com"
+EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[1]+'\r\n')
 
+EXAMPLE_EVENT3 = EXAMPLE_EVENT.copy()
+del EXAMPLE_EVENT3['source.ip']
+EXAMPLE_EVENT3['source.fqdn'] = "example.com"
+EXAMPLE_EVENT3["raw"] = utils.base64_encode(SAMPLE_SPLIT[2]+'\r\n')
+
+EXAMPLE_EVENT4 = EXAMPLE_EVENT.copy()
+EXAMPLE_EVENT4["raw"] = utils.base64_encode(SAMPLE_SPLIT[3]+'\r\n')
 
 class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase):
     """
@@ -45,13 +51,15 @@ def set_bot(cls):
                          "type": "malware",
                          "default_url_protocol": "http://"}
 
-    def test_error(self):
+    def test_invalid_value(self):
         """ Test if the error is raised. """
-        self.allowed_error_count = 1
-        self.run_bot()
+        self.run_bot(allowed_error_count=1)
+
         self.assertMessageEqual(0, EXAMPLE_EVENT)
+        self.assertMessageEqual(1, EXAMPLE_EVENT2)
+        self.assertMessageEqual(2, EXAMPLE_EVENT3)
         self.assertLogMatches("Failed to parse line.")
-        self.assertLogMatches("intelmq.lib.exceptions.InvalidValue: invalid value '-' \(<class 'str'>\) for key 'source.fqdn'")
+        self.assertLogMatches("intelmq.lib.exceptions.InvalidValue: invalid value '.' \(<class 'str'>\) for key 'source.fqdn'")
 
     def test_error_ignore(self):
         self.sysconfig = {"columns": ["time.source", "source.ip",
@@ -61,9 +69,11 @@ def test_error_ignore(self):
                           "default_url_protocol": "http://",
                           "columns_required": [True, True, False],
                           }
-        self.run_bot()
-        self.assertMessageEqual(0, EXAMPLE_EVENT2)
-        self.assertMessageEqual(1, EXAMPLE_EVENT)
+        self.run_bot(allowed_error_count=1)
+        self.assertMessageEqual(0, EXAMPLE_EVENT)
+        self.assertMessageEqual(1, EXAMPLE_EVENT2)
+        self.assertMessageEqual(2, EXAMPLE_EVENT3)
+        self.assertMessageEqual(3, EXAMPLE_EVENT4)
 
 
 if __name__ == '__main__':  # pragma: no cover