SNOW-94449: retry deletinig session when connection is closed

Author: sfc-gh-stakeda

connection.py

@@ -191,7 +191,7 @@ def __init__(self, **kwargs):
 
     def __del__(self):
         try:
-            self.close()
+            self.close(retry=False)
         except:
             pass
 
@@ -483,12 +483,13 @@ def connect(self, **kwargs):
         self.__set_error_attributes()
         self.__open_connection()
 
-    def close(self):
+    def close(self, retry=True):
         u"""
         Closes the connection.
         """
         try:
             if not self.rest:
+                logger.debug(u'Rest object has been destroyed, cannot close session')
                 return
 
             # will hang if the application doesn't close the connection and
@@ -499,12 +500,13 @@ def close(self):
             # close telemetry first, since it needs rest to send remaining data
             logger.info('closed')
             self._telemetry.close()
-            self.rest.delete_session()
+            self.rest.delete_session(retry=retry)
             self.rest.close()
             self._rest = None
             del self.messages[:]
-        except:
-            pass
+            logger.debug(u'Session is closed')
+        except Exception as e:
+            logger.debug(u'Exception encountered in closing connection. ignoring...: %s', e)
 
     def is_closed(self):
         u"""

network.py

@@ -429,7 +429,7 @@ def _heartbeat(self):
             logger.error("Failed to heartbeat. code: %s, url: %s",
                          ret.get(u'code'), url)
 
-    def delete_session(self):
+    def delete_session(self, retry=False):
         """
         Deletes the session
         """
@@ -452,19 +452,31 @@ def delete_session(self):
             headers[HTTP_HEADER_SERVICE_NAME] = self._connection.service_name
 
         body = {}
-        try:
-            ret = self._post_request(
-                url, headers, json.dumps(body),
-                token=self.token, timeout=5, no_retry=True)
-            if not ret or ret.get(u'success'):
-                return
-            err = ret.get(u'message')
-            if err is not None and ret.get(u'data'):
-                err += ret[u'data'].get(u'errorMessage', '')
-                # no exception is raised
-            logger.debug('error in deleting session. ignoring...: %s', err)
-        except Exception as e:
-            logger.debug('error in deleting session. ignoring...: %s', e)
+        retry_limit = 3 if retry else 1
+        num_retries = 0
+        should_retry = True
+        while should_retry and (num_retries < retry_limit):
+            try:
+                should_retry = False
+                ret = self._post_request(
+                    url, headers, json.dumps(body),
+                    token=self.token, timeout=5, no_retry=True)
+                if not ret:
+                    if retry:
+                        should_retry = True
+                    else:
+                        return
+                elif ret.get(u'success'):
+                    return
+                err = ret.get(u'message')
+                if err is not None and ret.get(u'data'):
+                    err += ret[u'data'].get(u'errorMessage', '')
+                    # no exception is raised
+                logger.debug('error in deleting session. ignoring...: %s', err)
+            except Exception as e:
+                logger.debug('error in deleting session. ignoring...: %s', e)
+            finally:
+                num_retries += 1
 
     def _get_request(self, url, headers, token=None,
                      timeout=None,