Merge pull request certtools#2495 from certtools/2493-shadowserver-api-bot

Handle type parameter as a string and match by type instead of filena…

Author: sebix

CHANGELOG.md

@@ -19,6 +19,9 @@
 
 ### Bots
 #### Collectors
+- `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
+  - Added support for the types parameter to be either a string or a list.
+  - Refactored to utilize the type field returned by the API to match the requested types instead of a sub-string match on the filename.
 - `intelmq.bots.collectors.shodan.collector_stream` (PR#2492 by Mikk Margus Möll):
   - Add `alert` parameter to Shodan stream collector to allow fetching streams by configured alert ID
 

intelmq/bots/collectors/shadowserver/collector_reports_api.py

@@ -59,7 +59,8 @@ def init(self):
             self._report_list = self.reports.split(',')
         elif isinstance(self.reports, list):
             self._report_list = self.reports
-
+        if isinstance(self.types, str):
+            self.types = self.types.split(',')
         if self.country and self.country not in self._report_list:
             self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.")
             self._report_list.append(self.country)
@@ -110,8 +111,7 @@ def _reports_list(self, date=None):
             return None
 
         if self.types:
-            reports = [report for report in reports if any(rtype in report['file'] for rtype in self.types)]
-
+            reports = [report for report in reports if any(report['type'] == rtype for rtype in self.types)]
         return reports
 
     def _report_download(self, reportid: str):

intelmq/tests/bots/collectors/shadowserver/reports-list.json

@@ -2,31 +2,36 @@
     {
         "report": "[email protected]",
         "file": "2020-08-02-scan_smb-anarres-geo.csv",
+	"type": "scan_smb",
         "id": "xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX",
         "timestamp": "2020-08-02"
     },
     {
         "report": "[email protected]",
         "id": "unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN",
         "file": "2020-08-02-cisco_smart_install-anarres-geo.csv",
+	"type": "cisco_smart_install",
         "timestamp": "2020-08-02"
     },
     {
         "timestamp": "2020-08-02",
         "id": "EhngMTvBT7tD4ehUpVJNqW8TRZRI9N6ozsarxuick4ritIIxOG",
         "file": "2020-08-02-scan_adb-anarres-geo.csv",
+	"type": "scan_adb",
         "report": "[email protected]"
     },
     {
         "id": "GYb7n9SbR5jM2PMsfvo78r3G7tYF4v37koXEB8Kngs3ewCvHF4",
         "file": "2020-08-02-scan_rsync-anarres-geo.csv",
+	"type": "scan_rsync",
         "report": "[email protected]",
         "timestamp": "2020-08-02"
     },
     {
         "timestamp": "2020-08-02",
         "report": "[email protected]",
         "file": "2020-08-02-scan_ldap_tcp-anarres-geo.csv",
+	"type": "scan_ldap_tcp",
         "id": "qxe9EGItMY7eyDQwPBwGgEP2VOpvZqnqSDRIJGkXy3UWVUC06B"
     }
 ]