Support extra_network and extra_subnet labels

This allows users to add extra network interface to cluster nodes, e.g.
storage network.

Story: 2002448
Task: 21983
Co-Authored-By: Bharat Kunwar <[email protected]>

Change-Id: I10a6a4d72e9ec635f2c73d9fe64a8d136228f532
(cherry picked from commit bea8c72)
(cherry picked from commit 8260f11)
(cherry picked from commit d6c0a2d)
(cherry picked from commit 6227555)
(cherry picked from commit 3667d28)
(cherry picked from commit 8562dde)

Author: mnasiadka

doc/source/user/index.rst

@@ -467,6 +467,12 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `octavia_lb_healthcheck`_             | see bellow         | true          |
 +---------------------------------------+--------------------+---------------+
+| `extra_network`_                      | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_subnet`_                       | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_security_group`_               | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 
 .. _cluster:
 
@@ -1553,6 +1559,22 @@ _`octavia_lb_healthcheck`
   If true, enable Octavia load balancer healthcheck
   Default: true
 
+_`extra_network`
+  Optional additional network name or UUID to add to cluster nodes.
+  When not specified, additional networks are not added. Optionally specify
+  'extra_subnet' if you wish to use a specific subnet on the network.
+  Default: ""
+
+_`extra_subnet`
+  Optional additional subnet name or UUID to add to cluster nodes.
+  Only used when 'extra_network' is defined.
+  Default: ""
+
+_`extra_security_group`
+  Optional additional group name or UUID to add to network port.
+  Only used when 'extra_network' is defined.
+  Default: cluster node default security group.
+
 Supported versions
 ------------------
 
@@ -2319,7 +2341,6 @@ _`calico_tag`
   Victoria default: v3.13.1
   Wallaby default: v3.13.1
 
-
 Besides, the Calico network driver needs kube_tag with v1.9.3 or later, because
 Calico needs extra mounts for the kubelet container. See `commit
 <https://github.com/projectatomic/atomic-system-containers/commit/54ab8abc7fa1bfb6fa674f55cd0c2fa0c812fd36>`_

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -123,7 +123,8 @@ def get_params(self, context, cluster_template, cluster, **kwargs):
                       'min_node_count', 'max_node_count', 'npd_enabled',
                       'ostree_remote', 'ostree_commit',
                       'use_podman', 'kube_image_digest',
-                      'metrics_scraper_tag']
+                      'metrics_scraper_tag',
+                      'extra_network', 'extra_subnet', 'extra_security_group']
 
         labels = self._get_relevant_labels(cluster, kwargs)
 

magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@@ -989,13 +989,32 @@ parameters:
     description: >
       Post install manifest URL used to setup some cloud provider/vendor
       specific configs
-    default: ""
+    default: ''
 
   master_lb_allowed_cidrs:
     type: comma_delimited_list
     description: The allowed CIDR list for master load balancer
     default: []
 
+  extra_network:
+    type: string
+    description: >
+      Additional network to bind nodes to
+    default: ''
+
+  extra_subnet:
+    type: string
+    description: >
+      Subnet for additional network
+    default: ''
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+    default: ''
+
+
 resources:
 
   ######################################################################
@@ -1384,6 +1403,9 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           post_install_manifest_url: {get_param: post_install_manifest_url}
           metrics_scraper_tag: {get_param: metrics_scraper_tag}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
 
   kube_cluster_config:
     condition: create_cluster_resources
@@ -1566,6 +1588,10 @@ resources:
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
           kube_service_account_key: {get_param: kube_service_account_key}
           kube_service_account_private_key: {get_param: kube_service_account_private_key}
+          extra_network: {get_param: extra_network}
+          extra_subnet: {get_param: extra_subnet}
+          extra_security_group: {get_param: extra_security_group}
+
 outputs:
 
   api_address:

magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml

@@ -698,6 +698,21 @@ parameters:
     description: >
       Tag of metrics-scraper for kubernetes dashboard.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional security group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -707,6 +722,24 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
 resources:
   ######################################################################
   #
@@ -965,7 +998,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -979,7 +1017,12 @@ resources:
       software_config_transport: POLL_SERVER_HEAT
       user_data: {get_resource: agent_config}
       networks:
-        - port: {get_resource: kube_master_eth0}
+        list_concat:
+          - - port: {get_resource: kube_master_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_master_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -998,6 +1041,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_master_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - {get_param: extra_security_group}
+            - {get_param: secgroup_kube_master_id}
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_master_floating:
     type: Magnum::Optional::KubeMaster::Neutron::FloatingIP
     properties:

magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml

@@ -387,6 +387,21 @@ parameters:
       The private key will be used to sign generated k8s service account
       tokens.
 
+  extra_network:
+    type: string
+    description: >
+      Additional network name to bind ports to instances
+
+  extra_subnet:
+    type: string
+    description: >
+      Additional subnet name
+
+  extra_security_group:
+    type: string
+    description: >
+      Additional seurity group name
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -396,6 +411,25 @@ conditions:
       - get_param: boot_volume_size
       - 0
 
+  extra_network_defined:
+    not:
+      equals:
+      - get_param: extra_network
+      - ''
+
+  extra_subnet_defined:
+    not:
+      equals:
+      - get_param: extra_subnet
+      - ''
+
+  extra_security_group_defined:
+    not:
+      equals:
+      - get_param: extra_security_group
+      - ''
+
+
 resources:
 
   agent_config:
@@ -563,7 +597,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
 
@@ -577,7 +616,12 @@ resources:
       user_data_format: SOFTWARE_CONFIG
       software_config_transport: POLL_SERVER_HEAT
       networks:
-        - port: {get_resource: kube_minion_eth0}
+        list_concat:
+          - - port: {get_resource: kube_minion_eth0}
+          - if:
+              - "extra_network_defined"
+              - - port: {get_resource: kube_minion_eth1}
+              - []
       scheduler_hints: { group: { get_param: nodes_server_group_id }}
       availability_zone: {get_param: availability_zone}
       block_device_mapping_v2:
@@ -596,6 +640,23 @@ resources:
         - ip_address: {get_param: pods_network_cidr}
       replacement_policy: AUTO
 
+  kube_minion_eth1:
+    type: OS::Neutron::Port
+    condition: extra_network_defined
+    properties:
+      network: {get_param: extra_network}
+      security_groups:
+        - if:
+            - "extra_security_group_defined"
+            - get_param: extra_security_group
+            - get_param: secgroup_kube_minion_id
+      fixed_ips:
+        if:
+          - "extra_subnet_defined"
+          - - subnet: {get_param: extra_subnet}
+          - []
+      replacement_policy: AUTO
+
   kube_minion_floating:
     type: Magnum::Optional::KubeMinion::Neutron::FloatingIP
     properties:

magnum/tests/unit/drivers/test_template_definition.py

@@ -553,6 +553,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
         octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm')
         octavia_lb_healthcheck = mock_cluster.labels.get(
                 'octavia_lb_healthcheck')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
 
         k8s_def = k8s_fcos_tdef.FCOSK8sTemplateDefinition()
 
@@ -676,6 +679,9 @@ def test_k8s_get_params(self, mock_generate_csr_and_key,
             'octavia_provider': octavia_provider,
             'octavia_lb_algorithm': octavia_lb_algorithm,
             'octavia_lb_healthcheck': octavia_lb_healthcheck,
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -1101,6 +1107,10 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
 
         master_lb_allowed_cidrs = mock_cluster.labels.get(
             'master_lb_allowed_cidrs')
+        extra_network = mock_cluster.labels.get('extra_network')
+        extra_subnet = mock_cluster.labels.get('extra_subnet')
+        extra_security_group = mock_cluster.labels.get('extra_security_group')
+
 
         octavia_provider = mock_cluster.labels.get('octavia_provider')
         octavia_lb_algorithm = mock_cluster.labels.get('octavia_lb_algorithm')
@@ -1231,6 +1241,9 @@ def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
             'octavia_provider': octavia_provider,
             'octavia_lb_algorithm': octavia_lb_algorithm,
             'octavia_lb_healthcheck': octavia_lb_healthcheck,
+            'extra_network': extra_network,
+            'extra_subnet': extra_subnet,
+            'extra_security_group': extra_security_group,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

releasenotes/notes/extra-network-interface-68bc169cb467a13e.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Support extra_network and extra_subnet labels to allow users to assign
+    extra network interface to cluster nodes, e.g. storage network. Also adds
+    possibility to set custom security group on network port.