Merge pull request #2374 from e3rd/sieve-time

sieve bot gets :before and :after keywords

Author: sebix

CHANGELOG.md

@@ -53,6 +53,8 @@ CHANGELOG
 - `intelmq.bots.experts.cymru_whois`:
   - Ignore AS names with unexpected unicode characters (PR#2352, fixes #2132)
   - Avoid extraneous search domain-based queries on NXDOMAIN result (PR#2352)
+- `intelmq.bots.experts.sieve`:
+  - Added :before and :after keywords (PR#2374)
 
 #### Outputs
 - `intelmq.bots.outputs.cif3.output`: Added (PR#2244 by Michael Davis).

docs/user/bots.rst

@@ -2194,7 +2194,7 @@ Both parameters accept string values describing absolute or relative time:
 
 * absolute
 
- * basically anything parseable by datetime parser, eg. "2015-09-012T06:22:11+00:00"
+ * basically anything parseable by datetime parser, eg. "2015-09-12T06:22:11+00:00"
  * `time.source` taken from the event will be compared to this value to decide the filter behavior
 
 * relative
@@ -2204,7 +2204,7 @@ Both parameters accept string values describing absolute or relative time:
 
 *Examples of time filter definition*
 
-* ```"not_before" : "2015-09-012T06:22:11+00:00"``` events older than the specified time will be dropped
+* ```"not_before" : "2015-09-12T06:22:11+00:00"``` events older than the specified time will be dropped
 * ```"not_after" : "6 months"``` just events older than 6 months will be passed through the pipeline
 
 **Possible paths**
@@ -3003,6 +3003,12 @@ The following operators may be used to match events:
  * `:supersetof` tests if the list of values from the given key is a superset of the values specified as the argument. Example for matching hosts with at least the IoT and vulnerable tags:
    ``if extra.tags :supersetof ['iot', 'vulnerable'] { ... }``
 
+ * `:before` tests if the date value occurred before given time ago. The time might be absolute (basically anything parseable by pendulum parser, eg. “2015-09-12T06:22:11+00:00”) or relative (accepted string formatted like this “<integer> <epoch>”, where epoch could be any of following strings (could optionally end with trailing ‘s’): hour, day, week, month, year)
+   ``if time.observation :before '1 week' { ... }``
+
+ * `:after`  tests if the date value occurred after given time ago; see `:before`
+    ``if time.observation :after '2015-09-12' { ... }  # happened after midnight the 12th Sep``
+
  * Boolean values can be matched with `==` or `!=` followed by `true` or `false`. Example:
    ``if extra.has_known_vulns == true { ... }``
 

intelmq/bots/experts/sieve/REQUIREMENTS.txt

@@ -1,4 +1,5 @@
 # SPDX-FileCopyrightText: 2017 Antoine Neuenschwander
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+pendulum
 textX>=1.5.1

intelmq/bots/experts/sieve/expert.py

@@ -13,28 +13,35 @@
 import os
 import re
 import traceback
-import datetime
 import operator
 
-from typing import Optional
+from datetime import datetime, timedelta, timezone
+from typing import Callable, Dict, Optional, Union
 from enum import Enum, auto
-from functools import partial
 
 import intelmq.lib.exceptions as exceptions
 from intelmq import HARMONIZATION_CONF_FILE
 from intelmq.lib import utils
 from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
+from intelmq.lib.message import Event
 from intelmq.lib.utils import parse_relative
 from intelmq.lib.harmonization import DateTime
 
+try:
+    from pendulum import parse
+except:
+    parse = None
+
 try:
     import textx.model
     from textx.metamodel import metamodel_from_file
     from textx.exceptions import TextXError, TextXSemanticError
 except ImportError:
     metamodel_from_file = None
 
+CondMap = Dict[str, Callable[['SieveExpertBot', object, Event], bool]]
+
 
 class Procedure(Enum):
     CONTINUE = auto()  # continue processing subsequent statements (default)
@@ -93,19 +100,28 @@ class SieveExpertBot(ExpertBot):
         '!=': operator.ne,
     }
 
-    _cond_map = {
+    _date_op_map = {
+        ':before': operator.lt,
+        ':after': operator.gt
+    }
+
+    _cond_map: CondMap = {
         'ExistMatch': lambda self, match, event: self.process_exist_match(match.key, match.op, event),
         'SingleStringMatch': lambda self, match, event: self.process_single_string_match(match.key, match.op, match.value, event),
         'MultiStringMatch': lambda self, match, event: self.process_multi_string_match(match.key, match.op, match.value, event),
         'SingleNumericMatch': lambda self, match, event: self.process_single_numeric_match(match.key, match.op, match.value, event),
         'MultiNumericMatch': lambda self, match, event: self.process_multi_numeric_match(match.key, match.op, match.value, event),
         'IpRangeMatch': lambda self, match, event: self.process_ip_range_match(match.key, match.range, event),
+        'DateMatch': lambda self, match, event: self.process_date_match(match.key, match.op, match.date, event),
         'ListMatch': lambda self, match, event: self.process_list_match(match.key, match.op, match.value, event),
         'BoolMatch': lambda self, match, event: self.process_bool_match(match.key, match.op, match.value, event),
         'Expression': lambda self, match, event: self.match_expression(match, event),
     }
 
     def init(self) -> None:
+        if parse is None:
+            raise MissingDependencyError("pendulum")
+
         if not SieveExpertBot._harmonization:
             harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE)
             SieveExpertBot._harmonization = harmonization_config['event']
@@ -300,6 +316,30 @@ def process_ip_range_match(self, key, ip_range, event) -> bool:
             return any(addr in ipaddress.ip_network(val.value, strict=False) for val in ip_range.values)
         raise TextXSemanticError(f'Unhandled type: {name}')
 
+    def parse_timeattr(self, time_attr) -> Union[datetime, timedelta]:
+        """ Parses relative or absolute time specification. """
+        try:
+            return parse(time_attr)
+        except ValueError:
+            return timedelta(minutes=parse_relative(time_attr))
+
+    def process_date_match(self, key, op, value, event) -> bool:
+        if key not in event:
+            return False
+
+        op = self._date_op_map[op]
+
+        base_time = self.parse_timeattr(value.value)
+        if isinstance(base_time, timedelta):
+            base_time = datetime.now(tz=timezone.utc) - base_time
+        try:
+            event_time = DateTime.from_isoformat(event[key], True)
+        except ValueError:
+            self.logger.warning("Could not parse %s=%s at %s.", key, event[key], event)
+            return False
+        else:
+            return op(event_time, base_time)
+
     def process_list_match(self, key, op, value, event) -> bool:
         if not (key in event and isinstance(event[key], list)):
             return False
@@ -316,7 +356,7 @@ def process_bool_match(self, key, op, value, event):
 
     def compute_basic_math(self, action, event) -> str:
         date = DateTime.from_isoformat(event[action.key], True)
-        delta = datetime.timedelta(minutes=parse_relative(action.value))
+        delta = timedelta(minutes=parse_relative(action.value))
 
         return self._basic_math_op_map[action.operator](date, delta).isoformat()
 

intelmq/bots/experts/sieve/sieve.tx

@@ -18,6 +18,7 @@ Condition:
         match=StringMatch
         | match=NumericMatch
         | match=IpRangeMatch
+        | match=DateMatch
         | match=ExistMatch
         | match=ListMatch
         | match=BoolMatch
@@ -68,6 +69,9 @@ IpRange: SingleIpRange | IpRangeList;
 SingleIpRange: value=STRING;
 IpRangeList: '[' values+=SingleIpRange[','] ']' ;
 
+DateMatch: key=Key op=DateOperator date=SingleStringValue;
+DateOperator: ':before' | ':after';
+
 ExistMatch: op=ExistOperator key=Key;
 ExistOperator: ':exists' | ':notexists';
 

intelmq/tests/bots/experts/sieve/test_expert.py

@@ -4,8 +4,10 @@
 
 # -*- coding: utf-8 -*-
 
+from pathlib import Path
 import unittest
 import os
+from datetime import timedelta, datetime
 import intelmq.lib.test as test
 from intelmq.bots.experts.sieve.expert import SieveExpertBot
 
@@ -949,6 +951,55 @@ def test_network_host_bits_list_match(self):
         self.run_bot()
         self.assertMessageEqual(0, event)
 
+    def test_date_match(self):
+        """ Test comparing absolute and relative to now dates. """
+        self.sysconfig['file'] = Path(__file__).parent / 'test_sieve_files/test_date_match.sieve'
+
+        def check(event, expected):
+            self.input_message = event
+            self.run_bot()
+            self.assertMessageEqual(0, expected)
+
+        event = EXAMPLE_INPUT.copy()
+        expected = event.copy()
+
+        event["time.observation"] = "2017-01-01T00:00:00+00:00"  # past event with tz
+        expected['extra.list'] = ['before 1 week', 'before 2023-06-01', 'before 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = "2017-01-01T00:00:00"  # past event without tz
+        check(event, expected)
+
+        event["time.observation"] = "2023-06-01"  # just date, neither before nor after the date's midnight
+        expected['extra.list'] = ['before 1 week', 'before 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = "2023-06-01 10:00"  # time given
+        expected['extra.list'] = ['before 1 week', 'after 2023-06-01', 'before 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = "2023-06-01T10:00+00:00"  # time including tz
+        check(event, expected)
+
+        event["time.observation"] = "2023-06-01T10:00-06:00"  # tz changes
+        expected['extra.list'] = ['before 1 week', 'after 2023-06-01', 'after 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = str(datetime.now())
+        expected['extra.list'] = ['after 1 week', 'after 2023-06-01', 'after 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = str(datetime.now() - timedelta(days=3))
+        check(event, expected)
+
+        event["time.observation"] = str(datetime.now() - timedelta(days=8))
+        expected['extra.list'] = ['before 1 week', 'after 2023-06-01', 'after 2023-06-01 15:00']
+        check(event, expected)
+
+        event["time.observation"] = str(datetime.now() - timedelta(days=8))
+        expected['extra.list'] = ['before 1 week', 'after 2023-06-01', 'after 2023-06-01 15:00']
+        check(event, expected)
+
     def test_comments(self):
         """ Test comments in sieve file."""
         self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_comments.sieve')
@@ -972,7 +1023,6 @@ def test_named_queues(self):
         self.run_bot()
         self.assertOutputQueueLen(0)
 
-
         # if doesn't match keep
         numeric_match_false = EXAMPLE_INPUT.copy()
         numeric_match_false['comment'] = "keep without path"
@@ -1301,7 +1351,6 @@ def test_bool_match(self):
         self.run_bot()
         self.assertMessageEqual(0, expected)
 
-
         # negative test with true == false
         event = base.copy()
         event['comment'] = 'match5'
@@ -1394,9 +1443,6 @@ def test_typed_values(self):
         self.run_bot()
         self.assertMessageEqual(0, expected)
 
-
-
-
     def test_append(self):
         ''' Test append action '''
         self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_append.sieve')

intelmq/tests/bots/experts/sieve/test_sieve_files/test_date_match.sieve

@@ -0,0 +1,28 @@
+if time.observation :before '1 week' {
+	append extra.list 'before 1 week'
+}
+
+if time.observation :after '1 week' {
+	append extra.list 'after 1 week'
+}
+
+if time.observation :before '2023-06-01' {
+	append extra.list 'before 2023-06-01'
+}
+
+if time.observation :after '2023-06-01' {
+	# when time is not set, '2023-06-01 00:01' resolves here
+	append extra.list 'after 2023-06-01'
+}
+
+if time.observation :before '2023-06-01 15:00' {
+	append extra.list 'before 2023-06-01 15:00'
+}
+
+if time.observation :after '2023-06-01 15:00' {
+	append extra.list 'after 2023-06-01 15:00'
+}
+
+if extra.str == 'few-hours' && time.observation :after '10 h' {
+	append extra.list 'after 10 h'
+}

intelmq/tests/bots/experts/sieve/test_sieve_files/test_date_match.sieve.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2023 Edvard Rejthar CSIRT.cz
+SPDX-License-Identifier: AGPL-3.0-or-later