Network find speedup (#747)

* Skip getting device's status if not necessay

This fix adds an additional flag that speeds up the network find
avoiding to always getting data from sqpoller to detect devices'
status.

Signed-off-by: Claudio Usai <[email protected]>

* Update tests, handle corner case for internal flag in device get

Signed-off-by: Claudio Usai <[email protected]>

* Update with suggestions

Signed-off-by: Claudio Usai <[email protected]>

* Update tests

Signed-off-by: Claudio Usai <[email protected]>

Author: claudious96

suzieq/engines/pandas/device.py

@@ -25,6 +25,7 @@ def get(self, **kwargs):
         status = kwargs.pop('status', '')
         os_version = kwargs.pop('version', '')
         os = kwargs.get('os', '')
+        ignore_neverpoll = kwargs.pop('ignore_neverpoll', False)
 
         addnl_fields = []
         fields = self.schema.get_display_fields(columns)
@@ -49,11 +50,13 @@ def get(self, **kwargs):
         if view == 'latest' and 'status' in df.columns:
             df['status'] = np.where(df.active, df['status'], 'dead')
 
-        poller_df = self._get_table_sqobj('sqPoller').get(
-            namespace=kwargs.get('namespace', []),
-            hostname=kwargs.get('hostname', []),
-            service='device',
-            columns='namespace hostname status timestamp'.split())
+        poller_df = pd.DataFrame()
+        if not ignore_neverpoll:
+            poller_df = self._get_table_sqobj('sqPoller').get(
+                namespace=kwargs.get('namespace', []),
+                hostname=kwargs.get('hostname', []),
+                service='device',
+                columns='namespace hostname status timestamp'.split())
 
         if not poller_df.empty:
             # Identify the address to namespace/hostname mapping

suzieq/engines/pandas/interfaces.py

@@ -544,7 +544,8 @@ def _add_portmode(self, df: pd.DataFrame, **kwargs):
 
         devdf = self._get_table_sqobj('device') \
             .get(namespace=namespace, hostname=hostname,
-                 columns=['namespace', 'hostname', 'os', 'status', 'vendor'])
+                 columns=['namespace', 'hostname', 'os'],
+                 ignore_neverpoll=True)
 
         pm_df = pd.DataFrame({'namespace': [], 'hostname': [],
                               'ifname': [], 'portmode': []})

suzieq/sqobjects/device.py

@@ -12,7 +12,7 @@ def __init__(self, **kwargs):
         super().__init__(table='device', **kwargs)
         self._valid_get_args = ['namespace', 'hostname', 'columns', 'os',
                                 'vendor', 'model', 'status', 'version',
-                                'query_str']
+                                'query_str', 'ignore_neverpoll']
         self._valid_arg_vals = {
             'status': ['alive', 'dead', 'neverpoll',
                        '!alive', '!dead', '!neverpoll']

tests/integration/test_rest.py

@@ -573,6 +573,9 @@ def test_rest_arg_consistency(service, verb):
 
         valid_args = set(arglist)
 
+        if service == 'device':
+            valid_args.remove('ignore_neverpoll')
+
         # In the tests below, we warn when we don't have the exact
         # {service}_{verb} REST function, which prevents us from picking the
         # correct set of args.
@@ -692,6 +695,8 @@ def test_routes_sqobj_consistency():
             args_to_match = get_args_to_match(sqobj, route.verbs)
             args_to_match = {a for a in args_to_match
                              if a not in common_args.union(top_args)}
+            if table == 'device':
+                args_to_match.remove('ignore_neverpoll')
             if args_to_match != query_params:
                 assert False, (f'different query params for {table}: expected '
                                f'{args_to_match}. Got {query_params}')