Merge pull request #869 from netenglabs/update-pandas

Update pandas

Author: ddutt

.pylintrc

@@ -763,5 +763,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

build/requirements.txt

@@ -23,8 +23,8 @@ textfsm = "*"
 async-timeout = "<4.0"
 aiofiles = "*"
 prompt-toolkit = ">2"
-pyarrow = "10.0.1"
-pandas = "1.3.4"
+pyarrow = "~11.0"
+pandas = "~1.5"
 PyYAML = "*"
 python-nubia = "0.2b5"
 uvloop = "*"
@@ -50,6 +50,7 @@ notebook = "6.4.12"
 urllib3 = "^1.26.12"
 packaging = "^21.3"
 psutil = "^5.9.4"
+jellyfish = "~0.10"
 
 [tool.poetry.dev-dependencies]
 pylint = "*"
@@ -59,15 +60,15 @@ jedi = "~0.17.2"
 flake8 = "*"
 ipdb = "*"
 pycscope = "*"
-pytest = "*"
+pytest = "~7.2"
 pytest-cov = "*"
 pytest-xdist = "*"
 pytest-asyncio = "*"
 pytest-mock = "*"
 mock = "*"
 pytest-depends = "^1.0.1"
 bandit = "^1.6.2"
-pytest-icdiff = "^0.5"
+pytest-icdiff = "~0.6"
 ipython = "~=7.16.3"
 requests = "^2.24.0"
 sphinx-autodoc-typehints = "^1.11.1"

poetry.lock

@@ -325,11 +325,15 @@ def _gen_output(self, df: pd.DataFrame, json_orient: str = "records",
                      or 'timestamp' in self.columns)):
                 if 'timestamp' in cols:
                     cols.remove('timestamp')
+
             with pd.option_context(
-                    'precision', 3,
+                    'format.precision', 3,
+                    'display.width', None,
+                    'display.max_columns', None,
                     'display.max_colwidth', max_colwidth,
-                    'display.max_rows', max_rows,
-                    'display.expand_frame_repr', not all_columns):
+                    'display.expand_frame_repr', not all_columns,
+                    'display.max_rows', max_rows):
+
                 df = self.sqobj.humanize_fields(df)
                 if df.empty:
                     print(df[cols])
@@ -382,7 +386,7 @@ def _invoke_sqobj(self, fn, **kwargs) -> pd.DataFrame:
                 df = pd.DataFrame({'error': [f'ERROR: UserQueryError: {ex}']})
             else:
                 if self.ctxt.debug:
-                    raise Exception from ex
+                    raise ex
                 df = pd.DataFrame({'error': [f'ERROR: {ex}']})
 
         return df

pyproject.toml

@@ -1,7 +1,7 @@
 
 from typing import Callable, Union
 import pandas as pd
-from pandas.core.computation.ops import UndefinedVariableError
+from pandas.errors import UndefinedVariableError
 from suzieq.shared.exceptions import SqVersConversionError
 from suzieq.shared.schema import Schema
 

suzieq/cli/sqcmds/command.py

@@ -587,7 +587,7 @@ def _post_summarize(self, check_empty_col: str = 'deviceCnt') -> None:
             if self.ns[ns][check_empty_col] == 0:
                 delete_keys.append(ns)
         for ns in delete_keys:
-            del(self.ns[ns])
+            del (self.ns[ns])
 
         ns_df = pd.DataFrame(self.ns)
         if len(self.summary_row_order) > 0:

suzieq/db/parquet/migratedb.py

@@ -291,11 +291,13 @@ def _find_fhr_df(self, device: str, ip: str) -> pd.DataFrame:
                 # is obtained from the MAC because of trunk ports.
                 if idf.empty:
                     continue
-                idf.at[idf.index, 'vlan'] = mac_entry.vlan.iloc[0]
+                idfidx = idf.index.values[0]
+                idf.at[idfidx, 'vlan'] = mac_entry.vlan.iloc[0]
+
                 if (idf.master == 'bridge').all():
-                    idf.at[idf.index, 'ipAddressList'] = f'{ip}/32'
+                    idf.at[idfidx, 'ipAddressList'] = f'{ip}/32'
                     # Assuming the VRF is identical across multiple entries
-                    idf.at[idf.index, 'master'] = rslt_df.iloc[0].vrf
+                    idf.at[idfidx, 'master'] = rslt_df.iloc[0].vrf
             else:
                 idf = self._if_df.query(f'hostname=="{row.hostname}" and '
                                         f'ifname=="{row.oif}" and '

suzieq/engines/pandas/engineobj.py

@@ -104,7 +104,8 @@ def _get_response(self, verb: str, **kwargs) -> pd.DataFrame:
             f'{verb}?'
             f'{query_params}')
 
-        response = requests.get(url, verify=False, )
+        # pylint: disable=missing-timeout
+        response = requests.get(url, verify=None)
         if response.status_code != 200:
             if response.text:
                 msg = response.json().get("detail", str(response.status_code))

suzieq/engines/pandas/path.py

@@ -128,7 +128,7 @@ def get_session_id():
     '''Return Streamlit's session ID'''
     ctx = get_script_run_ctx()
     if ctx is None:
-        raise Exception("Failed to get the thread context")
+        raise AttributeError("Failed to get the thread context")
     return ctx.session_id
 
 
@@ -166,8 +166,8 @@ def set_def_aggrid_options(grid_options: dict):
 
 def get_image_dir():
     '''Get directory where images are stored'''
-    return(os.path.dirname(find_spec('suzieq.gui')
-                           .loader.path) + '/images')
+    return (os.path.dirname(find_spec('suzieq.gui')
+                            .loader.path) + '/images')
 
 
 def color_row(row, **kwargs):