ENH: ignore tabs,spaces in new line or before comment

Author: kbrajneesh

intelmq/bots/parsers/generic/parser_csv.py

@@ -70,10 +70,12 @@ def init(self):
     def parse(self, report):
         raw_report = utils.base64_decode(report.get("raw"))
         raw_report = raw_report.translate({0: None})
-        # ignore lines starting with #
-        raw_report = re.sub(r'(?m)^#.*\n?', '', raw_report)
+        # ignore lines starting with #. # can have leading spaces/tabs
+        raw_report = re.sub(r'(?m)^[ \t]*#.*\n?', '', raw_report)
         # ignore null bytes
         raw_report = re.sub(r'(?m)\0', '', raw_report)
+        # ignore lines having mix of spaces and tabs only
+        raw_report = re.sub(r'(?m)^[ \t]*\n?', '', raw_report)
         # skip header
         if getattr(self.parameters, 'skip_header', False):
             self.tempdata.append(raw_report[:raw_report.find('\n')])

intelmq/tests/bots/parsers/generic/sample_report.csv

@@ -1,3 +1,12 @@
+ # comment with space before
+	# comment with tab before
+	  # comment with mix of tabs and spaces before
+ # below line contains spaces only
+   
+ # below line contains tabs only
+			
+ # below line contains mix of tabs and spaces only
+ 	  	
 # nesmysl jak noha
 2015-12-14 04:19:00	Testing	Really bad actor site comment	Nothing	Unimportant	www.cennoworld.com/Payment_Confirmation/Payment_Confirmation.zip	198.105.221.5	mail5.bulls.unisonplatform.com	just another comment
 2016-12-14 04:19:00	Testing	Really bad actor site comment	Nothing	Unimportant	www.cennoworld.com/Payment_Confirmation/Payment_Confirmation.zip	198.105.221.161	mail5.bulls.unisonplatform.com	just another comment

intelmq/tests/bots/parsers/generic/test_parser_csv.py

@@ -27,13 +27,13 @@
                  "source.fqdn": "mail5.bulls.unisonplatform.com",
                  "event_description.text": "Really bad actor site comment",
                  "classification.type": "malware",
-                 "raw": utils.base64_encode(SAMPLE_SPLIT[1].replace('\t', ',')+'\r\n'),
+                 "raw": utils.base64_encode(SAMPLE_SPLIT[10].replace('\t', ',')+'\r\n'),
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
 EXAMPLE_EVENT2 = EXAMPLE_EVENT.copy()
 EXAMPLE_EVENT2['time.source'] = "2016-12-14T04:19:00+00:00"
 EXAMPLE_EVENT2['source.ip'] = "198.105.221.161"
-EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[2].replace('\t', ',')+'\r\n')
+EXAMPLE_EVENT2["raw"] = utils.base64_encode(SAMPLE_SPLIT[11].replace('\t', ',')+'\r\n')
 
 
 class TestGenericCsvParserBot(test.BotTestCase, unittest.TestCase):