Merge branch 'develop' for 0.20.1

Author: ddutt

build/requirements.txt

@@ -86,6 +86,7 @@ prettytable==3.3.0 ; python_full_version > "3.8.1" and python_version < "3.10"
 prometheus-client==0.14.1 ; python_full_version > "3.8.1" and python_version < "3.10"
 prompt-toolkit==3.0.29 ; python_full_version > "3.8.1" and python_version < "3.10"
 protobuf==3.20.3 ; python_full_version > "3.8.1" and python_version < "3.10"
+psutil==5.9.4 ; python_full_version > "3.8.1" and python_version < "3.10"
 ptyprocess==0.7.0 ; python_full_version > "3.8.1" and python_version < "3.10" and os_name != "nt" or python_full_version > "3.8.1" and python_version < "3.10" and sys_platform != "win32"
 py==1.11.0 ; python_full_version > "3.8.1" and python_version < "3.10" and implementation_name == "pypy"
 pyarrow==10.0.1 ; python_full_version > "3.8.1" and python_version < "3.10"

docs/release-notes.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## 0.20.1 (Feb 22, 2023)
+
+This is a bugfix release. The main changes in this release including the bug fixes are:
+
+* Memory leak caused by one of the libraries we use has been eliminated by working around the leak in our code.
+* EOS interfaces that were in err-disabled or unconnected state were incorrectly marked as being admin down as well. Now, the admin state is shown correctly.
+* Displaying the processor and memory of the device on which the poller is running. This is displayed during the poller startup. A warning message is also printed if the server specifications are not optimal for running SuzieQ. This optimality is based on assumptions about the network such as devices, the amount of state per device etc. Individual user experience maybe different.
+* Reduced the number of outstanding commands per device to be always 1 for IOS/IOSXE. This is to avoid the outputs getting mixed up accidentally. This is a rare event, but nevertheless we want to eliminate it.
+
 ## 0.20.0 (Feb 2, 2023)
 
 This is largely a major bug fix release, fixing many parser issues, documentation updates. The main major addition is support for Junos EVO platforms. Among the major bug fixes are:

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "suzieq"
-version = "0.20.0"
+version = "0.20.1"
 description = "A framework and application for network observability"
 readme = 'README.md'
 repository = 'https://github.com/netenglabs/suzieq'
@@ -49,6 +49,7 @@ ciscoconfparse = "^1.6.21"
 notebook = "6.4.12"
 urllib3 = "^1.26.12"
 packaging = "^21.3"
+psutil = "^5.9.4"
 
 [tool.poetry.dev-dependencies]
 pylint = "*"

pyproject.toml

@@ -15,7 +15,8 @@
 from suzieq.shared.exceptions import InventorySourceError, PollingError, \
     SqPollerConfError
 from suzieq.shared.utils import (poller_log_params, init_logger,
-                                 load_sq_config, print_version)
+                                 load_sq_config, print_version,
+                                 log_suzieq_info)
 from suzieq.poller.controller.utils.inventory_utils import read_inventory
 
 
@@ -45,6 +46,8 @@ async def start_controller(user_args: argparse.Namespace, config_data: Dict):
             read_inventory(user_args.inventory)
             print('Inventory syntax check passed')
             return
+
+        log_suzieq_info('Poller Controller', logger, show_more=True)
         controller = Controller(user_args, config_data)
         controller.init()
         await controller.run()

suzieq/poller/sq_poller.py

@@ -528,10 +528,14 @@ def _set_devtype(self, devtype: str, version_str: str) -> None:
             elif self.devtype == "eos":
                 self.__class__ = EosNode
             elif self.devtype == "iosxe":
+                # Since we need to create one single interactive session with
+                # ios and iosxe, we cannot concurrently issue commands
+                self.batch_size = 1
                 self.__class__ = IosXENode
             elif self.devtype == "iosxr":
                 self.__class__ = IosXRNode
             elif self.devtype == "ios":
+                self.batch_size = 1
                 self.__class__ = IOSNode
             elif self.devtype.startswith("junos"):
                 self.__class__ = JunosNode

suzieq/poller/worker/nodes/node.py

@@ -2,13 +2,13 @@
 from datetime import datetime
 from copy import deepcopy
 
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (get_timestamp_from_cisco_time,
                                  get_timestamp_from_junos_time,
-                                 convert_asndot_to_asn)
+                                 convert_asndot_to_asn,
+                                 parse_relative_timestamp)
 
 
 # pylint: disable=too-many-statements
@@ -46,11 +46,8 @@ def _clean_eos_data(self, processed_data, raw_data):
                     entry['keepaliveTime'] = entry['configKeepalive']
                 continue
 
-            estdTime = parse(
-                entry.get('estdTime', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+            estdTime = parse_relative_timestamp(
+                entry.get('estdTime', ''), raw_data[0]['timestamp'], ms=True)
 
             if 'EVPN' in entry.get('afi', []):
                 afidx = entry['afi'].index('EVPN')
@@ -81,7 +78,7 @@ def _clean_eos_data(self, processed_data, raw_data):
                     new_entry['rrclient'] = True
                 else:
                     new_entry['rrclient'] = False
-                new_entry['estdTime'] = int(estdTime.timestamp()*1000)
+                new_entry['estdTime'] = estdTime
                 if entry['pfxBestRx']:
                     # Depending on the moodiness of the output, this field
                     # may not be present. So, ignore it.
@@ -448,13 +445,10 @@ def _clean_iosxr_data(self, processed_data, raw_data):
                 entry['afi'] = entry['afi'].lower()
             if entry.get('safi', ''):
                 entry['safi'] = entry['safi'].lower()
-            estdTime = parse(
-                entry.get('estdTime', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+            estdTime = parse_relative_timestamp(
+                entry.get('estdTime', ''), raw_data[0]['timestamp'], ms=True)
             if estdTime:
-                entry['estdTime'] = int(estdTime.timestamp()*1000)
+                entry['estdTime'] = estdTime
             entry['routerId'] = vrf_rtrid.get(entry['vrf'], '')
             if entry.get('rrclient', '') == '':
                 entry['rrclient'] = 'False'

suzieq/poller/worker/services/bgp.py

@@ -2,13 +2,13 @@
 from datetime import datetime
 from collections import defaultdict
 from json import loads
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (get_timestamp_from_junos_time,
-                                 expand_ios_ifname, expand_nxos_ifname)
-from suzieq.shared.utils import convert_macaddr_format_to_colon
+                                 expand_ios_ifname, expand_nxos_ifname,
+                                 convert_macaddr_format_to_colon,
+                                 parse_relative_timestamp)
 from suzieq.shared.utils import MISSING_SPEED, NO_SPEED, MISSING_SPEED_IF_TYPES
 
 
@@ -151,11 +151,9 @@ def _clean_eos_data(self, processed_data, _):
             adm_state = entry.get('adminState', 'down')
             if adm_state == 'notconnect':
                 entry['reason'] = 'notconnect'
-                entry['adminState'] = 'down'
                 entry['state'] = 'notConnected'
             elif adm_state == 'errdisabled':
                 entry['reason'] = 'errdisabled'
-                entry['adminState'] = 'down'
                 entry['state'] = 'errDisabled'
             elif adm_state == 'connected':
                 entry['adminState'] = 'up'
@@ -590,14 +588,10 @@ def fix_nxos_speed(entry):
                         if any(x in lastChange for x in 'dwmy'):
                             lastChange = f'{lastChange} hours ago'
 
-                        lastChange = parse(
-                            lastChange,
-                            settings={'RELATIVE_BASE': datetime.fromtimestamp(
-                                (raw_data[0]['timestamp'])/1000),
-                                'TIMEZONE': 'UTC'})
+                        lastChange = parse_relative_timestamp(
+                            lastChange, raw_data[0]['timestamp'], ms=True)
                     if lastChange:
-                        old_entry['statusChangeTimestamp'] = int(
-                            lastChange.timestamp() * 1000)
+                        old_entry['statusChangeTimestamp'] = lastChange
                     else:
                         old_entry['statusChangeTimestamp'] = 0
                     old_entry['description'] = entry.get('description', '')
@@ -895,14 +889,12 @@ def _clean_iosxr_data(self, processed_data, raw_data):
             entry['interfaceMac'] = convert_macaddr_format_to_colon(
                 entry.get('interfaceMac', '0000.0000.0000'))
 
-            lastChange = parse(
+            lastChange = parse_relative_timestamp(
                 entry.get('statusChangeTimestamp', ''),
-                settings={'RELATIVE_BASE':
-                          datetime.fromtimestamp(
-                              (raw_data[0]['timestamp'])/1000), })
+                raw_data[0]['timestamp'], ms=True)
+
             if lastChange:
-                entry['statusChangeTimestamp'] = int(lastChange.timestamp()
-                                                     * 1000)
+                entry['statusChangeTimestamp'] = lastChange
             if 'ipAddressList' not in entry:
                 entry['ipAddressList'] = []
                 entry['ip6AddressList'] = []

suzieq/poller/worker/services/interfaces.py

@@ -1,13 +1,11 @@
 import re
-from datetime import datetime
-
-from dateparser import parse
 import numpy as np
 
 from suzieq.poller.worker.services.service import Service
 from suzieq.shared.utils import (expand_nxos_ifname,
                                  get_timestamp_from_cisco_time,
-                                 get_timestamp_from_junos_time)
+                                 get_timestamp_from_junos_time,
+                                 parse_relative_timestamp)
 
 
 class RoutesService(Service):
@@ -289,11 +287,8 @@ def _clean_iosxr_data(self, processed_data, raw_data):
                     lastchange = lastchange.split(':')
                     lastchange = (f'{lastchange[0]} hour '
                                   f'{lastchange[1]}:{lastchange[2]} mins ago')
-                lastchange = parse(
-                    lastchange,
-                    settings={'RELATIVE_BASE':
-                              datetime.fromtimestamp(
-                                  (raw_data[0]['timestamp'])/1000), })
+                lastchange = parse_relative_timestamp(
+                    lastchange, raw_data[0]['timestamp'], ms=True)
             if lastchange:
                 entry['statusChangeTimestamp'] = lastchange.timestamp()*1000
             else:

suzieq/poller/worker/services/routes.py

@@ -8,10 +8,12 @@
 from typing import Dict
 
 import uvloop
+
 from suzieq.poller.worker.worker import Worker
 from suzieq.poller.worker.writers.output_worker import OutputWorker
 from suzieq.shared.exceptions import InventorySourceError, SqPollerConfError
-from suzieq.shared.utils import init_logger, load_sq_config, poller_log_params
+from suzieq.shared.utils import (init_logger, load_sq_config, log_suzieq_info,
+                                 poller_log_params)
 
 
 async def start_worker(userargs: argparse.Namespace, cfg: Dict):
@@ -29,6 +31,7 @@ async def start_worker(userargs: argparse.Namespace, cfg: Dict):
     logger = init_logger('suzieq.poller.worker', logfile,
                          loglevel, logsize, log_stdout)
 
+    log_suzieq_info('Poller Worker', logger)
     worker = None
     try:
         worker = Worker(userargs, cfg)