Merge pull request #2376 from kamil-certat/rt_filter_out

ENH: RTCollector: Exclude by subject and multiple values

Author: sebix

CHANGELOG.md

@@ -27,7 +27,10 @@ CHANGELOG
 ### Bots
 
 #### Collectors
-- `intelmq.bots.collector.rt`: restrict `python-rt` to be below version 3.0 due to introduced breaking changes.
+- `intelmq.bots.collector.rt`:
+  - restrict `python-rt` to be below version 3.0 due to introduced breaking changes,
+  - added support for `Subject NOT LIKE` queries,
+  - added support for multiple values in ticket subject queries.
 - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante).
 
 #### Parsers

docs/user/bots.rst

@@ -574,7 +574,8 @@ If none of the filename matches apply, the contents of the first (RT-) "history"
 * `search_queue`: queue of the ticket to search for (default: `Incident Reports`)
 * `search_requestor`: the e-mail address of the requestor
 * `search_status`: status of the ticket to search for (default: `new`)
-* `search_subject_like`: part of the subject of the ticket to search for (default: `Report`)
+* `search_subject_like`: part of the subject of the ticket to search for (default: `Report`); use list for multiple required values,
+* `search_subject_notlike`: exclude subject containing given value, use list for multiple excluding values,
 * `set_status`: status to set the ticket to after processing (default: `open`). `false` or `null` to not set a different status.
 * `take_ticket`: whether to take the ticket (default: `true`)
 * `url_regex`: regular expression of an URL to search for in the ticket

intelmq/bots/collectors/rt/collector_rt.py

@@ -35,6 +35,7 @@ class RTCollectorBot(CollectorBot, HttpMixin):
     search_requestor: Optional[str] = None
     search_status: str = "new"
     search_subject_like: str = "Report"
+    search_subject_not_like: str = None
     set_status: str = "open"
     ssl_client_certificate: str = None  # TODO: type should be pathlib.Path
     take_ticket: bool = True
@@ -47,8 +48,11 @@ class RTCollectorBot(CollectorBot, HttpMixin):
                          'search_requestor': 'Requestor',
                          'search_status': 'Status',
                          'search_subject_like': 'Subject__like',
+                         'search_subject_not_like': 'Subject__notlike',
                          }
 
+    RAW_QUERY_OP_MAPPING = {None: '=', 'like': ' LIKE ', 'notlike': ' NOT LIKE ', 'gt': '>'}
+
     def init(self):
         if rt is None:
             raise MissingDependencyError("rt")
@@ -85,10 +89,16 @@ def process(self):
         else:
             kwargs = {}
 
+        convert_to_raw = False
         for parameter_name, rt_name in self.PARAMETER_MAPPING.items():
             parameter_value = getattr(self, parameter_name, None)
             if parameter_value:
                 kwargs[rt_name] = parameter_value
+                if isinstance(parameter_value, list):
+                    convert_to_raw = True
+
+        if convert_to_raw:
+            kwargs = self._convert_to_raw_query(kwargs)
 
         query = RT.search(order='Created', **kwargs)
         self.logger.info('%s results on search query.', len(query))
@@ -205,5 +215,19 @@ def process(self):
             if self.set_status:
                 RT.edit_ticket(ticket_id, status=self.set_status)
 
+    @classmethod
+    def _convert_to_raw_query(cls, kwargs: dict) -> str:
+        parts = []
+        for key, values in kwargs.items():
+            if key == 'Queue':
+                continue
+            key_parts = key.split('__')
+            field_name, op = key_parts[0], None if len(key_parts) == 1 else key_parts[-1]
+            values = values if isinstance(values, list) else [values]
+            for value in values:
+                parts.append(f"({field_name}{cls.RAW_QUERY_OP_MAPPING[op]}\'{value}\')")
+
+        return {'Queue': kwargs.get('Queue'), 'raw_query': ' AND '.join(parts)}
+
 
 BOT = RTCollectorBot

intelmq/tests/bots/collectors/rt/test_collector.py

@@ -99,5 +99,45 @@ def test_url_zip(self):
         self.assertMessageEqual(1, REPORT_URL2)
 
 
+class TestRTRawQuery(unittest.TestCase):
+    def test_simple(self):
+        kwargs = {
+            'Status': 'active',
+            'Subject__like': 'Report',
+            'Subject__notlike': 'except',
+            'Created__gt': '2023-01-01',
+            'Queue': 'IR',
+        }
+
+        raw_query = RTCollectorBot._convert_to_raw_query(kwargs)
+
+        expected_query = ("(Status=\'active\')"
+                          " AND (Subject LIKE \'Report\')"
+                          " AND (Subject NOT LIKE \'except\')"
+                          " AND (Created>\'2023-01-01\')")
+
+        self.assertEqual(raw_query, {"Queue": "IR", "raw_query": expected_query})
+
+    def test_multiple_values(self):
+        kwargs = {
+            'Status': 'active',
+            'Subject__like': ['Report', 'Report 2'],
+            'Subject__notlike': ['except', 'except2'],
+            'Created__gt': '2023-01-01',
+            'Queue': 'IR',
+        }
+
+        raw_query = RTCollectorBot._convert_to_raw_query(kwargs)
+
+        expected_query = ("(Status=\'active\')"
+                          " AND (Subject LIKE \'Report\')"
+                          " AND (Subject LIKE \'Report 2\')"
+                          " AND (Subject NOT LIKE \'except\')"
+                          " AND (Subject NOT LIKE \'except2\')"
+                          " AND (Created>\'2023-01-01\')")
+
+        self.assertEqual(raw_query, {"Queue": "IR", "raw_query": expected_query})
+
+
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()