[OVN] Avoid deadlock when cleaning hash ring nodes

This patch avoids the clash of the hash ring cleaning operation and the
API workers by ensuring that the cleaning happens before the nodes for
that host are added to the ring and the connections to the OVSDBs (meaning
no events therefore no SELECTS on the hash ring table for that hostname).

This patch does this by re-using the same hash ring lock that starts
the probing thread. Now, the first worker that acquire the lock is
responsible for cleaning the hash ring for it's own host as well as
starting the probing thread. Subsequently workers only need to register
themselves to the hash ring.

Conflicts:
  neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py

Change-Id: Iba73f7944592a003232eb397ba1d4da3dcba5c3a
Closes-Bug: #1990174
Signed-off-by: Lucas Alvares Gomes <[email protected]>
(cherry picked from commit b7b8f7c)

Author: umago

neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py

@@ -117,10 +117,6 @@ def initialize(self):
         self.node_uuid = None
         self.hash_ring_group = ovn_const.HASH_RING_ML2_GROUP
         self.sg_enabled = ovn_acl.is_sg_enabled()
-        # NOTE(lucasagomes): _clean_hash_ring() must be called before
-        # self.subscribe() to avoid processes racing when adding or
-        # deleting nodes from the Hash Ring during service initialization
-        self._clean_hash_ring()
         self._post_fork_event = threading.Event()
         if cfg.CONF.SECURITYGROUP.firewall_driver:
             LOG.warning('Firewall driver configuration is ignored')
@@ -331,15 +327,29 @@ def should_post_fork_initialize(worker_class):
                                 service.RpcWorker)
 
     @lockutils.synchronized('hash_ring_probe_lock', external=True)
-    def _start_hash_ring_probe(self):
+    def _setup_hash_ring(self):
+        """Setup the hash ring.
+
+        The first worker to acquire the lock is responsible for cleaning
+        the hash ring from previous runs as well as start the probing
+        thread for this host. Subsequently workers just need to register
+        themselves to the hash ring.
+        """
+        admin_context = n_context.get_admin_context()
         if not self._hash_ring_probe_event.is_set():
+            self._clean_hash_ring()
+            self.node_uuid = ovn_hash_ring_db.add_node(admin_context,
+                                                       self.hash_ring_group)
             self._hash_ring_thread = maintenance.MaintenanceThread()
             self._hash_ring_thread.add_periodics(
                 maintenance.HashRingHealthCheckPeriodics(
                     self.hash_ring_group))
             self._hash_ring_thread.start()
             LOG.info("Hash Ring probing thread has started")
             self._hash_ring_probe_event.set()
+        else:
+            self.node_uuid = ovn_hash_ring_db.add_node(admin_context,
+                                                       self.hash_ring_group)
 
     def post_fork_initialize(self, resource, event, trigger, payload=None):
         # Initialize API/Maintenance workers with OVN IDL connections
@@ -351,10 +361,7 @@ def post_fork_initialize(self, resource, event, trigger, payload=None):
         self._ovn_client_inst = None
 
         if worker_class == neutron.wsgi.WorkerService:
-            admin_context = n_context.get_admin_context()
-            self.node_uuid = ovn_hash_ring_db.add_node(admin_context,
-                                                       self.hash_ring_group)
-            self._start_hash_ring_probe()
+            self._setup_hash_ring()
 
         n_agent.AgentCache(self)  # Initialize singleton agent cache
         self.nb_ovn, self.sb_ovn = impl_idl_ovn.get_ovn_idls(self, trigger)