sq-poller: fix backward compatibility with old data (+ upgrade pyarrow to 10.0.1) (#835)

* sq-poller: check if deviceSession is nan

Signed-off-by: Claudio Lorina <[email protected]>

* Bump pyarrow version up to 10.0.1

Signed-off-by: Claudio Lorina <[email protected]>

* sq-poller: add comment to deviceSession check

Signed-off-by: Claudio Lorina <[email protected]>

Signed-off-by: Claudio Lorina <[email protected]>

Author: claudiolor

poetry.lock

@@ -23,7 +23,7 @@ textfsm = "*"
 async-timeout = "<4.0"
 aiofiles = "*"
 prompt-toolkit = ">2"
-pyarrow = "8.0.0"
+pyarrow = "10.0.1"
 pandas = "1.3.4"
 PyYAML = "*"
 python-nubia = "0.2b5"

pyproject.toml

@@ -630,7 +630,16 @@ async def commit_data(self, result: Dict, namespace: str, hostname: str,
                 namespace=[namespace]).query('active')
             # Get the latest device session we have written with this service
             if not df.empty:
-                last_device_session = df['deviceSession'].iloc[0] or 0
+                # If we are dealing with old data then the value of
+                # deviceSession will be unset, if this is the case, set the
+                # value to 0 so that we force write. We need to check only
+                # the first value of deviceSession as it is equal for all the
+                # latest record, as in case of reboot we rewrite everything.
+                if not df['deviceSession'].iloc[:1].isnull().any():
+                    last_device_session = df['deviceSession'].iloc[0] or 0
+                else:
+                    last_device_session = 0
+
                 self._node_boot_timestamps[key] = int(last_device_session)
 
             prev_res = df.to_dict('records')