Merge pull request #100 from stackhpc/upstream/zed-2023-12-18

Synchronise zed with upstream

Author: markgoddard

neutron/db/db_base_plugin_v2.py

@@ -318,7 +318,7 @@ def _validate_projects_have_access_to_network(self, network, project_ids):
         if project_ids - allowed_projects:
             raise exc.InvalidSharedSetting(network=network.name)
 
-    def _validate_ipv6_attributes(self, subnet, cur_subnet):
+    def _validate_ipv6_attributes(self, subnet, cur_subnet, has_cidr):
         if cur_subnet:
             self._validate_ipv6_update_dhcp(subnet, cur_subnet)
             return
@@ -330,7 +330,7 @@ def _validate_ipv6_attributes(self, subnet, cur_subnet):
         if ra_mode_set and address_mode_set:
             self._validate_ipv6_combination(subnet['ipv6_ra_mode'],
                                             subnet['ipv6_address_mode'])
-        if address_mode_set or ra_mode_set:
+        if has_cidr and (address_mode_set or ra_mode_set):
             self._validate_eui64_applicable(subnet)
 
     def _validate_eui64_applicable(self, subnet):
@@ -607,16 +607,21 @@ def _validate_subnet(self, context, s, cur_subnet=None):
 
         ip_ver = s['ip_version']
 
+        # We could be called without a cidr if a subnet pool is being used,
+        # so remember that since some checks below require one and should
+        # be skipped.
+        has_cidr = False
         if validators.is_attr_set(s.get('cidr')):
             self._validate_ip_version(ip_ver, s['cidr'], 'cidr')
+            has_cidr = True
 
         # TODO(watanabe.isao): After we found a way to avoid the re-sync
         # from the agent side, this restriction could be removed.
         if cur_subnet:
             dhcp_was_enabled = cur_subnet.enable_dhcp
         else:
             dhcp_was_enabled = False
-        if s.get('enable_dhcp') and not dhcp_was_enabled:
+        if has_cidr and s.get('enable_dhcp') and not dhcp_was_enabled:
             subnet_prefixlen = netaddr.IPNetwork(s['cidr']).prefixlen
             error_message = _("Subnet has a prefix length that is "
                               "incompatible with DHCP service enabled")
@@ -640,12 +645,13 @@ def _validate_subnet(self, context, s, cur_subnet=None):
 
         if validators.is_attr_set(s.get('gateway_ip')):
             self._validate_ip_version(ip_ver, s['gateway_ip'], 'gateway_ip')
-            is_gateway_not_valid = (
-                ipam.utils.check_gateway_invalid_in_subnet(
-                    s['cidr'], s['gateway_ip']))
-            if is_gateway_not_valid:
-                error_message = _("Gateway is not valid on subnet")
-                raise exc.InvalidInput(error_message=error_message)
+            if has_cidr:
+                is_gateway_not_valid = (
+                    ipam.utils.check_gateway_invalid_in_subnet(
+                        s['cidr'], s['gateway_ip']))
+                if is_gateway_not_valid:
+                    error_message = _("Gateway is not valid on subnet")
+                    raise exc.InvalidInput(error_message=error_message)
             # Ensure the gateway IP is not assigned to any port
             # skip this check in case of create (s parameter won't have id)
             # NOTE(salv-orlando): There is slight chance of a race, when
@@ -698,7 +704,7 @@ def _validate_subnet(self, context, s, cur_subnet=None):
                     error_message=(_("ipv6_address_mode is not valid when "
                                      "ip_version is 4")))
         if ip_ver == 6:
-            self._validate_ipv6_attributes(s, cur_subnet)
+            self._validate_ipv6_attributes(s, cur_subnet, has_cidr)
 
     def _validate_subnet_for_pd(self, subnet):
         """Validates that subnet parameters are correct for IPv6 PD"""
@@ -860,6 +866,7 @@ def _create_subnet_precommit(self, context, subnet):
             msg = _('a subnetpool must be specified in the absence of a cidr')
             raise exc.BadRequest(resource='subnets', msg=msg)
 
+        validate = True
         if subnetpool_id:
             self.ipam.validate_pools_with_subnetpool(s)
             if subnetpool_id == constants.IPV6_PD_POOL_ID:
@@ -868,16 +875,18 @@ def _create_subnet_precommit(self, context, subnet):
                     # cidr with IPv6 prefix delegation. Set the subnetpool_id
                     # to None and allow the request to continue as normal.
                     subnetpool_id = None
-                    self._validate_subnet(context, s)
                 else:
                     prefix = constants.PROVISIONAL_IPV6_PD_PREFIX
                     subnet['subnet']['cidr'] = prefix
                     self._validate_subnet_for_pd(s)
+                    validate = False
         else:
             if not has_cidr:
                 msg = _('A cidr must be specified in the absence of a '
                         'subnet pool')
                 raise exc.BadRequest(resource='subnets', msg=msg)
+
+        if validate:
             self._validate_subnet(context, s)
 
         with db_api.CONTEXT_WRITER.using(context):

neutron/services/segments/db.py

@@ -31,6 +31,8 @@
 from oslo_log import log as logging
 from oslo_utils import uuidutils
 
+from neutron.db.models import agent as agent_model
+from neutron.db.models import segment as segment_model
 from neutron.db import segments_db as db
 from neutron import manager
 from neutron.objects import base as base_obj
@@ -243,14 +245,43 @@ def update_segment_host_mapping(context, host, current_segment_ids):
                               entry.segment_id, entry.host)
 
 
-def get_hosts_mapped_with_segments(context):
+def get_hosts_mapped_with_segments(context, include_agent_types=None,
+                                   exclude_agent_types=None):
     """Get hosts that are mapped with segments.
 
     L2 providers can use this method to get an overview of SegmentHostMapping,
     and then delete the stale SegmentHostMapping.
+
+    When using both include_agent_types and exclude_agent_types,
+    exclude_agent_types is most significant.
+    All hosts without agent are excluded when using any agent_type filter.
+
+    :param context: current running context information
+    :param include_agent_types: (set) List of agent types, include hosts
+        with matching agents.
+    :param exclude_agent_types: (set) List of agent types, exclude hosts
+        with matching agents.
     """
-    segment_host_mapping = network.SegmentHostMapping.get_objects(context)
-    return {row.host for row in segment_host_mapping}
+    def add_filter_by_agent_types(qry, include, exclude):
+        qry = qry.join(
+            agent_model.Agent,
+            segment_model.SegmentHostMapping.host == agent_model.Agent.host)
+        if include:
+            qry = qry.filter(agent_model.Agent.agent_type.in_(include))
+        if exclude:
+            qry = qry.filter(agent_model.Agent.agent_type.not_in(exclude))
+
+        return qry
+
+    with db_api.CONTEXT_READER.using(context):
+        query = context.session.query(segment_model.SegmentHostMapping)
+        if include_agent_types or exclude_agent_types:
+            query = add_filter_by_agent_types(query, include_agent_types,
+                                              exclude_agent_types)
+
+        res = query.all()
+
+    return {row.host for row in res}
 
 
 def _get_phys_nets(agent):

neutron/tests/unit/db/test_db_base_plugin_v2.py

@@ -6657,6 +6657,47 @@ def test_allocate_any_ipv4_subnet_ipv6_pool(self):
             res = req.get_response(self.api)
             self.assertEqual(400, res.status_int)
 
+    def _test_allocate_any_subnet_wrong_dns(self, pool_cidr, ip_version,
+                                            nameservers):
+        with self.network() as network:
+            sp = self._test_create_subnetpool([pool_cidr],
+                                              tenant_id=self._tenant_id,
+                                              name=self._POOL_NAME)
+
+            # Specify a DNS nameserver of the wrong IP version
+            data = {'subnet': {'network_id': network['network']['id'],
+                               'subnetpool_id': sp['subnetpool']['id'],
+                               'ip_version': ip_version,
+                               'dns_nameservers': nameservers,
+                               'tenant_id': network['network']['tenant_id']}}
+            req = self.new_create_request('subnets', data)
+            res = req.get_response(self.api)
+            self.assertEqual(400, res.status_int)
+
+    def test_allocate_any_v4_subnet_wrong_dns_v6(self):
+        self._test_allocate_any_subnet_wrong_dns('10.0.0.0/16',
+            constants.IP_VERSION_4, ['2001:db8:1:2::1'])
+
+    def test_allocate_any_v6_subnet_wrong_dns_v4(self):
+        self._test_allocate_any_subnet_wrong_dns('2001:db8:1:2::/63',
+            constants.IP_VERSION_6, ['10.0.0.1'])
+
+    def test_allocate_any_v4_subnet_wrong_dns_v6_multiple(self):
+        self._test_allocate_any_subnet_wrong_dns('10.0.0.0/16',
+            constants.IP_VERSION_4, ['2001:db8:1:2::1', '2001:db8:2:2::1'])
+
+    def test_allocate_any_v6_subnet_wrong_dns_v4_multiple(self):
+        self._test_allocate_any_subnet_wrong_dns('2001:db8:1:2::/63',
+            constants.IP_VERSION_6, ['11.0.0.11', '12.0.0.12'])
+
+    def test_allocate_any_v4_subnet_wrong_dns_mixed(self):
+        self._test_allocate_any_subnet_wrong_dns('10.0.0.0/16',
+            constants.IP_VERSION_4, ['11.0.0.11', '2001:db8:1:2::1'])
+
+    def test_allocate_any_v6_subnet_wrong_dns_mixed(self):
+        self._test_allocate_any_subnet_wrong_dns('2001:db8:1:2::/63',
+            constants.IP_VERSION_6, ['11.0.0.11', '2001:db8:1:2::1'])
+
 
 class DbModelMixin(object):
     """DB model tests."""

neutron/tests/unit/extensions/test_segment.py

@@ -748,10 +748,37 @@ def test_get_all_hosts_mapped_with_segments(self):
         actual_hosts = db.get_hosts_mapped_with_segments(ctx)
         self.assertEqual(hosts, actual_hosts)
 
+    def test_get_all_hosts_mapped_with_segments_agent_type_filter(self):
+        ctx = context.get_admin_context()
+        hosts = set()
+        with self.network() as network:
+            network_id = network['network']['id']
+        for i in range(1, 3):
+            host = "host%s" % i
+            segment = self._test_create_segment(
+                network_id=network_id, physical_network='physnet%s' % i,
+                segmentation_id=200 + i, network_type=constants.TYPE_VLAN)
+            db.update_segment_host_mapping(
+                ctx, host, {segment['segment']['id']})
+            hosts.add(host)
+
+        # Now they are 2 hosts with segment being mapped.
+        #  host1 does not have an agent
+        #  host2 does not have an agent
+        # Any agent_type filter excludes hosts that does not have an agent
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, exclude_agent_types={'fake-agent-type'})
+        self.assertEqual(set(), actual_hosts)
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, include_agent_types={'fake-agent-type'})
+        self.assertEqual(set(), actual_hosts)
+
 
 class TestMl2HostSegmentMappingOVS(HostSegmentMappingTestCase):
     _mechanism_drivers = ['openvswitch', 'logger']
     mock_path = 'neutron.services.segments.db.update_segment_host_mapping'
+    agent_type_a = constants.AGENT_TYPE_OVS
+    agent_type_b = constants.AGENT_TYPE_LINUXBRIDGE
 
     def test_new_agent(self):
         host = 'host1'
@@ -871,9 +898,118 @@ def test_agent_with_no_mappings(self, mock):
         self.assertFalse(segments_host_db)
         self.assertFalse(mock.mock_calls)
 
+    def test_get_all_hosts_mapped_with_segments(self):
+        ctx = context.get_admin_context()
+        hosts = set()
+        with self.network() as network:
+            network_id = network['network']['id']
+        for i in range(1, 3):
+            host = "host%s" % i
+            segment = self._test_create_segment(
+                network_id=network_id, physical_network='physnet%s' % i,
+                segmentation_id=200 + i, network_type=constants.TYPE_VLAN)
+            self._register_agent(host, mappings={'physnet%s' % i: 'br-eth-1'},
+                                 plugin=self.plugin)
+            db.update_segment_host_mapping(
+                ctx, host, {segment['segment']['id']})
+            hosts.add(host)
+
+        # Now they are 2 hosts with segment being mapped.
+        actual_hosts = db.get_hosts_mapped_with_segments(ctx)
+        self.assertEqual(hosts, actual_hosts)
+
+    def test_get_all_hosts_mapped_with_segments_agent_type_filters(self):
+        ctx = context.get_admin_context()
+        with self.network() as network:
+            network_id = network['network']['id']
+        for i in range(1, 3):
+            host = "host%s" % i
+            segment = self._test_create_segment(
+                network_id=network_id, physical_network='physnet%s' % i,
+                segmentation_id=200 + i, network_type=constants.TYPE_VLAN)
+            if i == 2:
+                agent_type = self.agent_type_a
+            else:
+                agent_type = self.agent_type_b
+            helpers.register_ovs_agent(
+                host, agent_type=agent_type,
+                bridge_mappings={'physnet%s' % i: 'br-eth-1'},
+                plugin=self.plugin, start_flag=True)
+            db.update_segment_host_mapping(
+                ctx, host, {segment['segment']['id']})
+
+        # Now they are 2 hosts with segment being mapped.
+        #   host1 is agent_type_b
+        #   host2 is agent_type_a
+        # get all hosts (host1 and host2) when not using any filtering
+        actual_hosts = db.get_hosts_mapped_with_segments(ctx)
+        self.assertEqual({"host1", "host2"}, actual_hosts)
+        # get host1 when exclude agent_type_a agents
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, exclude_agent_types={self.agent_type_a})
+        self.assertEqual({"host1"}, actual_hosts)
+        # get host2 when exclude agent_type_b agents
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, exclude_agent_types={self.agent_type_b})
+        self.assertEqual({"host2"}, actual_hosts)
+        # get host2 when include agent_type_a agents
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, include_agent_types={self.agent_type_a})
+        self.assertEqual({"host2"}, actual_hosts)
+        # get host1 when include agent_type_b agents
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, include_agent_types={self.agent_type_b})
+        self.assertEqual({"host1"}, actual_hosts)
+        # get host1 and host2 when include both agent_type_a and agent_type_b
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, include_agent_types={self.agent_type_b, self.agent_type_a})
+        self.assertEqual({"host1", "host2"}, actual_hosts)
+        # When using both include and exclude, exclude is most significant
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx,
+            include_agent_types={self.agent_type_b, self.agent_type_a},
+            exclude_agent_types={self.agent_type_b}
+        )
+        self.assertEqual({"host2"}, actual_hosts)
+        # include and exclude both agent types - exclude is most significant
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx,
+            include_agent_types={self.agent_type_b, self.agent_type_a},
+            exclude_agent_types={self.agent_type_b, self.agent_type_a}
+        )
+        self.assertEqual(set(), actual_hosts)
+
+    def test_get_all_hosts_mapped_with_segments_agent_type_filter(self):
+        ctx = context.get_admin_context()
+        hosts = set()
+        with self.network() as network:
+            network_id = network['network']['id']
+        for i in range(1, 3):
+            host = "host%s" % i
+            segment = self._test_create_segment(
+                network_id=network_id, physical_network='physnet%s' % i,
+                segmentation_id=200 + i, network_type=constants.TYPE_VLAN)
+            self._register_agent(host, mappings={'physnet%s' % i: 'br-eth-1'},
+                                 plugin=self.plugin)
+            db.update_segment_host_mapping(
+                ctx, host, {segment['segment']['id']})
+            hosts.add(host)
+
+        # Now they are 2 hosts with segment being mapped.
+        #  host1 is agent_type_a
+        #  host2 is agent_type_a
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, exclude_agent_types={self.agent_type_a})
+        self.assertEqual(set(), actual_hosts)
+        actual_hosts = db.get_hosts_mapped_with_segments(
+            ctx, include_agent_types={self.agent_type_a})
+        self.assertEqual(hosts, actual_hosts)
+
 
 class TestMl2HostSegmentMappingLinuxBridge(TestMl2HostSegmentMappingOVS):
     _mechanism_drivers = ['linuxbridge', 'logger']
+    agent_type_a = constants.AGENT_TYPE_LINUXBRIDGE
+    agent_type_b = constants.AGENT_TYPE_OVS
 
     def setUp(self, plugin=None):
         cfg.CONF.set_override(c_experimental.EXPERIMENTAL_LINUXBRIDGE, True,
@@ -888,6 +1024,8 @@ def _register_agent(self, host, mappings=None, plugin=None):
 
 class TestMl2HostSegmentMappingMacvtap(TestMl2HostSegmentMappingOVS):
     _mechanism_drivers = ['macvtap', 'logger']
+    agent_type_a = constants.AGENT_TYPE_MACVTAP
+    agent_type_b = constants.AGENT_TYPE_OVS
 
     def _register_agent(self, host, mappings=None, plugin=None):
         helpers.register_macvtap_agent(host=host, interface_mappings=mappings,
@@ -896,6 +1034,8 @@ def _register_agent(self, host, mappings=None, plugin=None):
 
 class TestMl2HostSegmentMappingSriovNicSwitch(TestMl2HostSegmentMappingOVS):
     _mechanism_drivers = ['sriovnicswitch', 'logger']
+    agent_type_a = constants.AGENT_TYPE_NIC_SWITCH
+    agent_type_b = constants.AGENT_TYPE_OVS
 
     def _register_agent(self, host, mappings=None, plugin=None):
         helpers.register_sriovnicswitch_agent(host=host,