Retry connections to Nova

Sometimes Neutron is failing to send notification to Nova
due to timeout, refused connection or another HTTP error.
Retry send in those cases.

Closes-Bug: #1987780
Change-Id: Iaaccec770484234b704f70f3c144efac4d8ffba0
(cherry picked from commit cd475f9)

Author: Szymon Wroblewski

neutron/notifiers/nova.py

@@ -16,6 +16,7 @@
 import contextlib
 
 from eventlet.green import threading
+from keystoneauth1 import exceptions as ks_exceptions
 from keystoneauth1 import loading as ks_loading
 from neutron_lib.callbacks import events
 from neutron_lib.callbacks import registry
@@ -32,6 +33,7 @@
 from oslo_log import log as logging
 from oslo_utils import uuidutils
 from sqlalchemy.orm import attributes as sql_attr
+import tenacity
 
 from neutron.notifiers import batch_notifier
 
@@ -267,6 +269,12 @@ def notify_port_active_direct(self, port):
              'tag': port.id})
         self.send_port_status(None, None, port)
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            ks_exceptions.RetriableConnectionFailure),
+        wait=tenacity.wait_exponential(multiplier=0.01, max=1),
+        stop=tenacity.stop_after_delay(1),
+        after=tenacity.after_log(LOG, logging.DEBUG))
     def send_events(self, batched_events):
         LOG.debug("Sending events: %s", batched_events)
         novaclient = self._get_nova_client()
@@ -276,6 +284,10 @@ def send_events(self, batched_events):
         except nova_exceptions.NotFound:
             LOG.debug("Nova returned NotFound for event: %s",
                       batched_events)
+        except ks_exceptions.RetriableConnectionFailure:
+            raise
+            # next clause handles all exceptions
+            # so reraise for retry decorator
         except Exception:
             LOG.exception("Failed to notify nova on events: %s",
                           batched_events)

neutron/tests/unit/notifiers/test_nova.py

@@ -17,6 +17,7 @@
 from unittest import mock
 
 import eventlet
+from keystoneauth1 import exceptions as ks_exc
 from neutron_lib import constants as n_const
 from neutron_lib import context as n_ctx
 from neutron_lib import exceptions as n_exc
@@ -248,6 +249,14 @@ def test_nova_send_event_rasies_404(self, mock_client):
             nova_exceptions.NotFound)
         self.nova_notifier.send_events([])
 
+    @mock.patch('novaclient.client.Client')
+    def test_nova_send_events_raises_connect_exc(self, mock_client):
+        create = mock_client().server_external_events.create
+        create.side_effect = (
+            ks_exc.ConnectFailure, ks_exc.ConnectTimeout, [])
+        self.nova_notifier.send_events([])
+        self.assertEqual(3, create.call_count)
+
     @mock.patch('novaclient.client.Client')
     def test_nova_send_events_raises(self, mock_client):
         mock_client.server_external_events.create.return_value = Exception