Merge pull request #856 from netenglabs/dateparser-leak

sq-poller: remove RELATIVE_BASE option to avoid leak

Author: ddutt

suzieq/poller/worker/services/bgp.py

@@ -2,13 +2,13 @@
 from datetime import datetime
 from copy import deepcopy
 
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (get_timestamp_from_cisco_time,
                                  get_timestamp_from_junos_time,
-                                 convert_asndot_to_asn)
+                                 convert_asndot_to_asn,
+                                 parse_relative_timestamp)
 
 
 # pylint: disable=too-many-statements
@@ -46,11 +46,8 @@ def _clean_eos_data(self, processed_data, raw_data):
                     entry['keepaliveTime'] = entry['configKeepalive']
                 continue
 
-            estdTime = parse(
-                entry.get('estdTime', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+            estdTime = parse_relative_timestamp(
+                entry.get('estdTime', ''), raw_data[0]['timestamp'], ms=True)
 
             if 'EVPN' in entry.get('afi', []):
                 afidx = entry['afi'].index('EVPN')
@@ -81,7 +78,7 @@ def _clean_eos_data(self, processed_data, raw_data):
                     new_entry['rrclient'] = True
                 else:
                     new_entry['rrclient'] = False
-                new_entry['estdTime'] = int(estdTime.timestamp()*1000)
+                new_entry['estdTime'] = estdTime
                 if entry['pfxBestRx']:
                     # Depending on the moodiness of the output, this field
                     # may not be present. So, ignore it.
@@ -448,13 +445,10 @@ def _clean_iosxr_data(self, processed_data, raw_data):
                 entry['afi'] = entry['afi'].lower()
             if entry.get('safi', ''):
                 entry['safi'] = entry['safi'].lower()
-            estdTime = parse(
-                entry.get('estdTime', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+            estdTime = parse_relative_timestamp(
+                entry.get('estdTime', ''), raw_data[0]['timestamp'], ms=True)
             if estdTime:
-                entry['estdTime'] = int(estdTime.timestamp()*1000)
+                entry['estdTime'] = estdTime
             entry['routerId'] = vrf_rtrid.get(entry['vrf'], '')
             if entry.get('rrclient', '') == '':
                 entry['rrclient'] = 'False'

suzieq/poller/worker/services/interfaces.py

@@ -2,13 +2,13 @@
 from datetime import datetime
 from collections import defaultdict
 from json import loads
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (get_timestamp_from_junos_time,
-                                 expand_ios_ifname, expand_nxos_ifname)
-from suzieq.shared.utils import convert_macaddr_format_to_colon
+                                 expand_ios_ifname, expand_nxos_ifname,
+                                 convert_macaddr_format_to_colon,
+                                 parse_relative_timestamp)
 from suzieq.shared.utils import MISSING_SPEED, NO_SPEED, MISSING_SPEED_IF_TYPES
 
 
@@ -590,14 +590,10 @@ def fix_nxos_speed(entry):
                         if any(x in lastChange for x in 'dwmy'):
                             lastChange = f'{lastChange} hours ago'
 
-                        lastChange = parse(
-                            lastChange,
-                            settings={'RELATIVE_BASE': datetime.fromtimestamp(
-                                (raw_data[0]['timestamp'])/1000),
-                                'TIMEZONE': 'UTC'})
+                        lastChange = parse_relative_timestamp(
+                            lastChange, raw_data[0]['timestamp'], ms=True)
                     if lastChange:
-                        old_entry['statusChangeTimestamp'] = int(
-                            lastChange.timestamp() * 1000)
+                        old_entry['statusChangeTimestamp'] = lastChange
                     else:
                         old_entry['statusChangeTimestamp'] = 0
                     old_entry['description'] = entry.get('description', '')
@@ -895,14 +891,12 @@ def _clean_iosxr_data(self, processed_data, raw_data):
             entry['interfaceMac'] = convert_macaddr_format_to_colon(
                 entry.get('interfaceMac', '0000.0000.0000'))
 
-            lastChange = parse(
+            lastChange = parse_relative_timestamp(
                 entry.get('statusChangeTimestamp', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+                raw_data[0]['timestamp'], ms=True)
+
             if lastChange:
-                entry['statusChangeTimestamp'] = int(lastChange.timestamp()
-                                                     * 1000)
+                entry['statusChangeTimestamp'] = lastChange
             if 'ipAddressList' not in entry:
                 entry['ipAddressList'] = []
                 entry['ip6AddressList'] = []

suzieq/poller/worker/services/routes.py

@@ -1,13 +1,11 @@
 import re
-from datetime import datetime
-
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (expand_nxos_ifname,
                                  get_timestamp_from_cisco_time,
-                                 get_timestamp_from_junos_time)
+                                 get_timestamp_from_junos_time,
+                                 parse_relative_timestamp)
 
 
 class RoutesService(Service):
@@ -289,11 +287,8 @@ def _clean_iosxr_data(self, processed_data, raw_data):
                     lastchange = lastchange.split(':')
                     lastchange = (f'{lastchange[0]} hour '
                                   f'{lastchange[1]}:{lastchange[2]} mins ago')
-                lastchange = parse(
-                    lastchange,
-                    settings={'RELATIVE_BASE':
-                              datetime.fromtimestamp(
-                                  (raw_data[0]['timestamp'])/1000), })
+                lastchange = parse_relative_timestamp(
+                    lastchange, raw_data[0]['timestamp'], ms=True)
             if lastchange:
                 entry['statusChangeTimestamp'] = lastchange.timestamp()*1000
             else:

suzieq/shared/utils.py

@@ -6,13 +6,14 @@
 import os
 import re
 import sys
-from datetime import datetime, timezone
+from datetime import datetime
 from enum import Enum
 from importlib.util import find_spec
 from ipaddress import ip_network
 from itertools import groupby
 from logging.handlers import RotatingFileHandler
 from os import getenv
+from time import time
 from typing import Any, Dict, List, Tuple
 from tzlocal import get_localzone
 
@@ -381,41 +382,61 @@ def calc_avg(oldval, newval):
     return float((oldval+newval)/2)
 
 
-def parse_relative_timestamp(uptime: str, relative_to: int = None) -> int:
+def parse_relative_timestamp(
+        uptime: str, relative_to: int = None, ms=False) -> int:
     """Get a relative time (i.e. with format 10 weeks, 4 days, 3 hours 11 mins)
     and convert it into a timestamp.
 
     Args:
         uptime (str): _description_
         relative_to (int, optional): provide a custom base epoch timestamp, if
-            not provided, the base timestamp is "now".
+            not provided, the base timestamp is "now". Time in s or ms
+            depending on the value of the `ms` argument.
+        ms (bool, optional): whether the function assumes time in seconds or
+            milliseconds. Default to False.
 
     Returns:
-        int: The epoch timestamp of the base time minus the uptime
+        int: The epoch timestamp of the base time minus the uptime. Returned
+            time in ms or s depending if ms=True
     """
+    # We do not use the RELATIVE_BASE option of dateparser due to a bug in the
+    # library causing a memory leak:
+    # https://github.com/scrapinghub/dateparser/issues/985
+
     settings = {'TIMEZONE': 'utc',
                 'RETURN_AS_TIMEZONE_AWARE': True}
-    if relative_to:
-        base_ts = datetime.fromtimestamp(relative_to, timezone.utc)
-        settings['RELATIVE_BASE'] = base_ts
+    conversion_value = 1000 if ms else 1
+
+    parsed_uptime = parse(uptime, settings=settings)
+    if not parsed_uptime:
+        return None
+
+    ts_offset = time() - (relative_to / conversion_value) if relative_to else 0
 
-    return int(parse(uptime, settings=settings).timestamp())
+    return int((parsed_uptime.timestamp() - ts_offset) * conversion_value)
 
 
-def get_timestamp_from_cisco_time(in_data, timestamp) -> int:
+def get_timestamp_from_cisco_time(in_data: str, timestamp: int) -> int:
     """Get timestamp in ms from the Cisco-specific timestamp string
     Examples of Cisco timestamp str are P2DT14H45M16S, P1M17DT4H49M50S etc.
+
+    Args:
+        in_data (str): The Cisco uptime string
+        timestamp (int): the base unix timestamp IS SECONDS from which
+            subtracting the uptime.
+
+    Returns:
+        int: a unix timestamp in milliseconds
     """
     if in_data and not in_data.startswith('P'):
         in_data = in_data.replace('y', 'years')
         in_data = in_data.replace('w', 'weeks')
         in_data = in_data.replace('d', 'days')
 
-        other_time = parse(in_data,
-                           settings={'RELATIVE_BASE':
-                                     datetime.utcfromtimestamp(timestamp)})
+        ts_offset = time() - timestamp
+        other_time = parse(in_data)
         if other_time:
-            return int(other_time.timestamp()*1000)
+            return int((other_time.timestamp() - ts_offset) * 1000)
         else:
             logger.error(f'Unable to parse relative time string, {in_data}')
             return 0