[cuegui] Handle gRPC CANCELLED errors to fix UI refresh issues (AcademySoftwareFoundation#2042)

**Link the Issue(s) this Pull Request is related to.**
- AcademySoftwareFoundation#2041

**Summarize your change.**

Fix UI refresh failures caused by unhandled gRPC CANCELLED and
UNAVAILABLE errors during connection interruptions. The UI would stop
updating after performing actions like "eat and mark done" or "retry",
requiring a restart.

Changes:
- Add CancelledException class and map StatusCode.CANCELLED to exception
- Update ThreadPool to gracefully handle gRPC connection errors with
warning instead of error, allowing automatic retry on next update cycle
- Add gRPC error handling to FrameMonitorTree._getUpdate() and
_getUpdateChanged() to recover from transient connection issues
- Return None on CANCELLED/UNAVAILABLE to trigger full update on retry
- Return empty list on NOT_FOUND to clear deleted jobs from view

This allows the UI to automatically recover from transient network or
server issues without stopping background updates or requiring manual
restart.

---------

Signed-off-by: Ramon Figueiredo <ramon.fgrd@gmail.com>

Author: ramonfigueiredo

cuegui/cuegui/FrameMonitorTree.py

@@ -33,6 +33,7 @@
 from qtpy import QtCore
 from qtpy import QtGui
 from qtpy import QtWidgets
+import grpc
 
 import opencue
 from opencue_proto import job_pb2
@@ -484,8 +485,19 @@ def _getUpdate(self):
                 self.__lastUpdateTime = int(time.time())
                 return self.__job.getFrames(**self.frameSearch.options)
             return []
+        except grpc.RpcError as e:
+            # Handle gRPC errors - log but don't crash, allow UI to retry
+            # pylint: disable=no-member
+            if hasattr(e, 'code') and e.code() in [grpc.StatusCode.CANCELLED,
+                                                     grpc.StatusCode.UNAVAILABLE]:
+                logger.warning("gRPC connection interrupted during frame update, will retry")
+            else:
+                logger.error("gRPC error in _getUpdate: %s", e)
+            # pylint: enable=no-member
+            return []
         except opencue.exception.CueException as e:
             list(map(logger.warning, cuegui.Utils.exceptionOutput(e)))
+            return []
 
     def _getUpdateChanged(self):
         """Returns the updated data from the cuebot
@@ -504,8 +516,24 @@ def _getUpdateChanged(self):
             self.__jobState = updated_data.state
             updatedFrames = updated_data.updated_frames.updated_frames
 
+        except grpc.RpcError as e:
+            # Handle gRPC errors - allow UI to continue and retry
+            # pylint: disable=no-member
+            if hasattr(e, 'code'):
+                if e.code() in [grpc.StatusCode.CANCELLED, grpc.StatusCode.UNAVAILABLE]:
+                    logger.warning("gRPC connection interrupted during frame update, will retry")
+                    # Return None to trigger a full update on next cycle
+                    return None
+                if e.code() == grpc.StatusCode.NOT_FOUND:
+                    # Job was deleted
+                    logger.info("Job not found, clearing job from view")
+                    self.setJob(None)
+                    return []
+                logger.error("gRPC error in _getUpdateChanged: %s", e)
+            # pylint: enable=no-member
+            return None
         except opencue.EntityNotFoundException:
-            self.setJobObj(None)
+            self.setJob(None)
         except opencue.exception.CueException as e:
             # pylint: disable=no-member
             if hasattr(e, "message") and 'timestamp cannot be over a minute off' in e.message:

cuegui/cuegui/ThreadPool.py

@@ -50,6 +50,7 @@ def someWorkCallback(work, result):
 import os
 
 from qtpy import QtCore
+import grpc
 
 import cuegui.Logger
 
@@ -213,6 +214,16 @@ def run(self):
                     if work[1]:
                         self.workComplete.emit(work, result)
                         del result
+                except grpc.RpcError as e:
+                    # Handle gRPC errors gracefully - these are often transient
+                    # pylint: disable=no-member
+                    if hasattr(e, 'code') and e.code() in [grpc.StatusCode.CANCELLED,
+                                                             grpc.StatusCode.UNAVAILABLE]:
+                        logger.warning("gRPC connection issue for '%s': %s - "
+                                     "UI will retry on next update", work[2], e.details())
+                    else:
+                        logger.error("gRPC error processing work for '%s': %s", work[2], e)
+                    # pylint: enable=no-member
                 except TypeError as e:
                     logger.error("TypeError in work processing for '%s': %s", work[2], e)
                 except ValueError as e:

pycue/opencue/exception.py

@@ -62,6 +62,13 @@ class ConnectionException(CueException):
     retryable = True
 
 
+class CancelledException(CueException):
+    """Raised when the RPC call was cancelled."""
+    failMsg = 'RPC call was cancelled. {details}'
+    retryMsg = 'RPC call was cancelled, retrying...'
+    retryable = True
+
+
 def getRetryCount():
     """Return the configured number of retries a cuebot call can make.
     If not specified in the config, all retryable calls will be called once and retried 3 times."""
@@ -73,5 +80,6 @@ def getRetryCount():
     grpc.StatusCode.ALREADY_EXISTS: EntityAlreadyExistsException,
     grpc.StatusCode.DEADLINE_EXCEEDED: DeadlineExceededException,
     grpc.StatusCode.INTERNAL: CueInternalErrorException,
-    grpc.StatusCode.UNAVAILABLE: ConnectionException
+    grpc.StatusCode.UNAVAILABLE: ConnectionException,
+    grpc.StatusCode.CANCELLED: CancelledException
 }