Merge pull request #122 from stackhpc/upstream/2023.1-2024-02-12

Synchronise 2023.1 with upstream

Author: markgoddard

doc/source/contributor/internals/openvswitch_agent.rst

@@ -494,7 +494,7 @@ The IDs used for bridge and port names are truncated.
          |          tbr-trunk-id          |
          |                                |
          | tpt-parent-id   spt-subport-id |
-         |                   (tag 100)    |
+         |  (tag 0)          (tag 100)    |
          +-----+-----------------+--------+
                |                 |
                |                 |
@@ -514,27 +514,27 @@ spi-subport-id: int bridge side of the patch port that implements a subport.
 Trunk creation
 ++++++++++++++
 
-A VM is spawned passing to Nova the port-id of a parent port associated with
-a trunk. Neutron will pass to Nova the bridge where to plug the vif as part of the vif details.
-The os-vif driver creates the trunk bridge tbr-trunk-id if it does not exist in plug().
-It will create the tap interface tap1 and plug it into tbr-trunk-id setting the parent port ID in the external-ids.
-The OVS agent will be monitoring the creation of ports on the trunk bridges. When it detects
-that a new port has been created on the trunk bridge, it will do the following:
+A VM is spawned passing to Nova the port-id of a parent port associated
+with a trunk. Neutron will pass to Nova the bridge where to plug the
+vif as part of the vif details.  The os-vif driver creates the trunk
+bridge tbr-trunk-id if it does not exist in plug().  It will create the
+tap interface tap1 and plug it into tbr-trunk-id setting the parent port
+ID in the external-ids.  The trunk driver will wire the parent port via
+a patch port to connect the trunk bridge to the integration bridge:
 
 ::
 
- ovs-vsctl add-port tbr-trunk-id tpt-parent-id -- set Interface tpt-parent-id type=patch options:peer=tpi-parent-id
- ovs-vsctl add-port br-int tpi-parent-id tag=3 -- set Interface tpi-parent-id type=patch options:peer=tpt-parent-id
+ ovs-vsctl add-port tbr-trunk-id tpt-parent-id -- set Interface tpt-parent-id type=patch options:peer=tpi-parent-id -- set Port tpt-parent-id vlan_mode=access tag=0
+ ovs-vsctl add-port br-int tpi-parent-id -- set Interface tpi-parent-id type=patch options:peer=tpt-parent-id
 
 
-A patch port is created to connect the trunk bridge to the integration bridge.
-tpt-parent-id, the trunk bridge side of the patch is not associated to any
-tag. It will carry untagged traffic.
-tpi-parent-id, the br-int side the patch port is tagged with VLAN 3. We assume that the
-trunk is on network1 that on this host is associated with VLAN 3.
-The OVS agent will set the trunk ID in the external-ids of tpt-parent-id and tpi-parent-id.
-If the parent port is associated with one or more subports the agent will process them as
-described in the next paragraph.
+tpt-parent-id, the trunk bridge side of the patch will carry untagged
+traffic (vlan_mode=access tag=0).  The OVS agent will be monitoring the
+creation of ports on the integration bridge.  tpi-parent-id, the br-int
+side the patch port is tagged with VLAN 3 by ovs-agent.  We assume that
+the trunk is on network1 that on this host is associated with VLAN 3.
+If the parent port is associated with one or more subports the agent
+will process them as described in the next paragraph.
 
 Subport creation
 ++++++++++++++++

neutron/agent/common/base_agent_rpc.py

@@ -0,0 +1,33 @@
+# Copyright 2024 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron_lib import rpc as n_rpc
+from oslo_messaging import Target
+
+
+class BasePluginApi(object):
+    """Base agent side of the rpc API"""
+    def __init__(self, topic, namespace, version):
+        target = Target(
+            topic=topic,
+            namespace=namespace,
+            version=version)
+        self.client = n_rpc.get_client(target)
+
+    def get_ports(self, context, port_filters):
+        # NOTE(mtomaska): The MetadataRpcCallback (server side) API version 1.0
+        # exposes get_ports, under the PLUGIN topic and None namespace.
+        cctxt = self.client.prepare(version='1.0')
+        return cctxt.call(context, 'get_ports', filters=port_filters)

neutron/agent/dhcp/agent.py

@@ -25,7 +25,6 @@
 from neutron_lib import constants
 from neutron_lib import context
 from neutron_lib import exceptions
-from neutron_lib import rpc as n_rpc
 from oslo_concurrency import lockutils
 from oslo_config import cfg
 from oslo_log import helpers as log_helpers
@@ -38,6 +37,7 @@
 from oslo_utils import timeutils
 
 from neutron._i18n import _
+from neutron.agent.common import base_agent_rpc
 from neutron.agent.common import resource_processing_queue as queue
 from neutron.agent.linux import dhcp
 from neutron.agent.linux import external_process
@@ -838,7 +838,7 @@ def disable_isolated_metadata_proxy(self, network):
             del self._metadata_routers[network.id]
 
 
-class DhcpPluginApi(object):
+class DhcpPluginApi(base_agent_rpc.BasePluginApi):
     """Agent side of the dhcp rpc API.
 
     This class implements the client side of an rpc interface.  The server side
@@ -858,11 +858,10 @@ class DhcpPluginApi(object):
 
     def __init__(self, topic, host):
         self.host = host
-        target = oslo_messaging.Target(
+        super().__init__(
             topic=topic,
             namespace=constants.RPC_NAMESPACE_DHCP_PLUGIN,
             version='1.0')
-        self.client = n_rpc.get_client(target)
 
     @property
     def context(self):
@@ -918,6 +917,11 @@ def get_dhcp_port(self, port_id):
         if port:
             return dhcp.DictModel(port)
 
+    def get_ports(self, port_filters):
+        ports = super().get_ports(self.context, port_filters)
+        if ports:
+            return [dhcp.DictModel(port) for port in ports]
+
     def dhcp_ready_on_ports(self, port_ids):
         """Notify the server that DHCP is configured for the port."""
         cctxt = self.client.prepare(version='1.5')

neutron/agent/linux/dhcp.py

@@ -357,8 +357,8 @@ def _get_process_manager(self, cmd_callback=None):
 
     def disable(self, retain_port=False, block=False, **kwargs):
         """Disable DHCP for this network by killing the local process."""
-        self.process_monitor.unregister(self.network.id, DNSMASQ_SERVICE_NAME)
         pm = self._get_process_manager()
+        self.process_monitor.unregister(pm.uuid, DNSMASQ_SERVICE_NAME)
         pm.disable(sig=str(int(signal.SIGTERM)))
         if block:
             try:
@@ -602,7 +602,7 @@ def _spawn_or_reload_process(self, reload_with_HUP):
 
         pm.enable(reload_cfg=reload_with_HUP, ensure_active=True)
 
-        self.process_monitor.register(uuid=self.get_process_uuid(),
+        self.process_monitor.register(uuid=pm.uuid,
                                       service_name=DNSMASQ_SERVICE_NAME,
                                       monitored_process=pm)
 
@@ -1210,11 +1210,15 @@ def _output_opts_file(self):
         return name
 
     def _get_ovn_metadata_port_ip(self, subnet):
-        m_ports = [port for port in self.network.ports if
-                   self._is_ovn_metadata_port(port, self.network.id)]
-        if m_ports:
-            port = self.device_manager.plugin.get_dhcp_port(m_ports[0].id)
-            for fixed_ip in port.fixed_ips:
+        """Check if provided subnet contains OVN metadata port"""
+        ports_result = self.device_manager.plugin.get_ports(
+            port_filters={
+                'device_owner': [constants.DEVICE_OWNER_DISTRIBUTED],
+                'device_id': ['ovnmeta-' + self.network.id]
+            },
+        )
+        if ports_result:
+            for fixed_ip in ports_result[0].get('fixed_ips', []):
                 if fixed_ip.subnet_id == subnet.id:
                     return fixed_ip.ip_address
 

neutron/agent/linux/interface.py

@@ -374,11 +374,6 @@ def _add_device_to_namespace(self, ip_wrapper, device, namespace):
                 LOG.warning("Failed to set interface %s into namespace %s. "
                             "Interface not found, attempt: %s, retrying.",
                             device, namespace, i + 1)
-                # NOTE(slaweq) In such case it's required to reset device's
-                # namespace as it was already set to the "namespace"
-                # and after retry neutron will look for it in that namespace
-                # which is wrong
-                device.namespace = None
                 time.sleep(1)
             except utils.WaitTimeout:
                 # NOTE(slaweq): if the exception was WaitTimeout then it means

neutron/agent/linux/ip_lib.py

@@ -25,6 +25,7 @@
 from neutron_lib import exceptions
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_utils import excutils
 from oslo_utils import netutils
 from pyroute2.netlink import exceptions as netlink_exceptions
 from pyroute2.netlink import rtnl
@@ -477,16 +478,16 @@ def set_down(self):
             self.name, self._parent.namespace, state='down')
 
     def set_netns(self, namespace, is_ovs_port=False):
-        privileged.set_link_attribute(
-            self.name, self._parent.namespace, net_ns_fd=namespace)
-        self._parent.namespace = namespace
-        if is_ovs_port:
-            # NOTE(slaweq): because of the "shy port" which may dissapear for
-            # short time after it's moved to the namespace we need to wait
-            # a bit before checking if port really exists in the namespace
-            time.sleep(1)
-        common_utils.wait_until_true(lambda: self.exists, timeout=5,
-                                     sleep=0.5)
+        old_namespace = self._parent.namespace
+        try:
+            privileged.set_link_attribute(
+                self.name, self._parent.namespace, net_ns_fd=namespace)
+            self._parent.namespace = namespace
+            common_utils.wait_until_true(lambda: self.exists, timeout=3,
+                                         sleep=0.5)
+        except common_utils.WaitTimeout:
+            with excutils.save_and_reraise_exception():
+                self._parent.namespace = old_namespace
 
     def set_name(self, name):
         privileged.set_link_attribute(

neutron/agent/metadata/agent.py

@@ -19,17 +19,16 @@
 from neutron_lib.agent import topics
 from neutron_lib import constants
 from neutron_lib import context
-from neutron_lib import rpc as n_rpc
 from neutron_lib.utils import host
 from oslo_config import cfg
 from oslo_log import log as logging
-import oslo_messaging
 from oslo_service import loopingcall
 from oslo_utils import netutils
 import requests
 import webob
 
 from neutron._i18n import _
+from neutron.agent.common import base_agent_rpc
 from neutron.agent.linux import utils as agent_utils
 from neutron.agent import rpc as agent_rpc
 from neutron.common import cache_utils as cache
@@ -46,7 +45,7 @@
 }
 
 
-class MetadataPluginAPI(object):
+class MetadataPluginAPI(base_agent_rpc.BasePluginApi):
     """Agent-side RPC for metadata agent-to-plugin interaction.
 
     This class implements the client side of an rpc interface used by the
@@ -61,15 +60,10 @@ class MetadataPluginAPI(object):
     """
 
     def __init__(self, topic):
-        target = oslo_messaging.Target(
+        super().__init__(
             topic=topic,
             namespace=constants.RPC_NAMESPACE_METADATA,
             version='1.0')
-        self.client = n_rpc.get_client(target)
-
-    def get_ports(self, context, filters):
-        cctxt = self.client.prepare()
-        return cctxt.call(context, 'get_ports', filters=filters)
 
 
 class MetadataProxyHandler(object):

neutron/db/l3_db.py

@@ -1889,7 +1889,7 @@ def _make_floatingip_dict_with_scope(self, floatingip_obj, scope_id):
         d['fixed_ip_address_scope'] = scope_id
         return d
 
-    def _get_sync_floating_ips(self, context, router_ids):
+    def _get_sync_floating_ips(self, context, router_ids, host=None):
         """Query floating_ips that relate to list of router_ids with scope.
 
         This is different than the regular get_floatingips in that it finds the
@@ -1905,7 +1905,7 @@ def _get_sync_floating_ips(self, context, router_ids):
         return [
             self._make_floatingip_dict_with_scope(*scoped_fip)
             for scoped_fip in l3_obj.FloatingIP.get_scoped_floating_ips(
-                context, router_ids)
+                context, router_ids, host)
         ]
 
     def _get_sync_interfaces(self, context, router_ids, device_owners=None):
@@ -2055,7 +2055,7 @@ def _process_interfaces(self, routers_dict, interfaces):
                 router[constants.INTERFACE_KEY] = router_interfaces
 
     def _get_router_info_list(self, context, router_ids=None, active=None,
-                              device_owners=None):
+                              device_owners=None, fip_host_filter=None):
         """Query routers and their related floating_ips, interfaces."""
         with db_api.CONTEXT_WRITER.using(context):
             routers = self._get_sync_routers(context,
@@ -2064,7 +2064,8 @@ def _get_router_info_list(self, context, router_ids=None, active=None,
             router_ids = [router['id'] for router in routers]
             interfaces = self._get_sync_interfaces(
                 context, router_ids, device_owners)
-            floating_ips = self._get_sync_floating_ips(context, router_ids)
+            floating_ips = self._get_sync_floating_ips(
+                context, router_ids, host=fip_host_filter)
             return (routers, interfaces, floating_ips)
 
     def get_sync_data(self, context, router_ids=None, active=None):

neutron/db/l3_dvr_db.py

@@ -979,9 +979,17 @@ def _get_fip_agent_gw_ports(self, context, fip_agent_id):
     @log_helper.log_method_call
     def _get_dvr_sync_data(self, context, host, agent, router_ids=None,
                            active=None):
+        # If the requesting agent is in normal dvr mode, we can fetch
+        # only FIPs bound to the particular host requesting the update
+        requesting_agent_mode = self._get_agent_mode(agent)
+        fip_host_filter = None
+        if requesting_agent_mode == const.L3_AGENT_MODE_DVR:
+            fip_host_filter = host
+
         routers, interfaces, floating_ips = self._get_router_info_list(
             context, router_ids=router_ids, active=active,
-            device_owners=const.ROUTER_INTERFACE_OWNERS)
+            device_owners=const.ROUTER_INTERFACE_OWNERS,
+            fip_host_filter=fip_host_filter)
         dvr_router_ids = set(router['id'] for router in routers
                              if is_distributed_router(router))
         floating_ip_port_ids = [fip['port_id'] for fip in floating_ips
@@ -1014,7 +1022,6 @@ def _get_dvr_sync_data(self, context, host, agent, router_ids=None,
                     if len(l3_agent_on_host):
                         l3_agent_mode = self._get_agent_mode(
                             l3_agent_on_host[0])
-                    requesting_agent_mode = self._get_agent_mode(agent)
                     # Consider the ports where the portbinding host and
                     # request host match.
                     if port_host == host:

neutron/objects/router.py

@@ -21,6 +21,7 @@
 from oslo_utils import versionutils
 from oslo_versionedobjects import fields as obj_fields
 from sqlalchemy import func
+from sqlalchemy import or_
 from sqlalchemy import sql
 
 from neutron.db.models import dvr as dvr_models
@@ -30,6 +31,7 @@
 from neutron.db import models_v2
 from neutron.objects import base
 from neutron.objects.qos import binding as qos_binding
+from neutron.plugins.ml2 import models as ml2_models
 
 
 @base.NeutronObjectRegistry.register
@@ -390,7 +392,7 @@ def obj_make_compatible(self, primitive, target_version):
 
     @classmethod
     @db_api.CONTEXT_READER
-    def get_scoped_floating_ips(cls, context, router_ids):
+    def get_scoped_floating_ips(cls, context, router_ids, host=None):
         query = context.session.query(l3.FloatingIP,
                                       models_v2.SubnetPool.address_scope_id)
         query = query.join(
@@ -405,11 +407,28 @@ def get_scoped_floating_ips(cls, context, router_ids):
             models_v2.SubnetPool,
             models_v2.Subnet.subnetpool_id == models_v2.SubnetPool.id)
 
+        # If a host value is provided, filter output to a specific host
+        if host is not None:
+            query = query.outerjoin(
+                ml2_models.PortBinding,
+                models_v2.Port.id == ml2_models.PortBinding.port_id)
+            # Also filter for ports with migrating_to as they may be relevant
+            # to this host but might not yet have the 'host' column updated
+            # if the migration is in a pre-live migration state
+            query = query.filter(or_(
+                ml2_models.PortBinding.host == host,
+                ml2_models.PortBinding.profile.like('%migrating_to%'),
+            ))
+
         # Filter out on router_ids
         query = query.filter(l3.FloatingIP.router_id.in_(router_ids))
 
-        # Remove duplicate rows based on FIP IDs
-        query = query.group_by(l3.FloatingIP.id)
+        # Remove duplicate rows based on FIP IDs and the subnet pool address
+        # scope. Only one subnet pool (per IP version, 4 in this case) can
+        # be assigned to a subnet. The subnet pool address scope for a FIP is
+        # unique.
+        query = query.group_by(l3.FloatingIP.id,
+                               models_v2.SubnetPool.address_scope_id)
 
         for row in query:
             yield (cls._load_object(context, row[0]), row[1])