Use the same bridge for the mgmt network and external connectivity (#216)

Author: robertvolkmann

Makefile

@@ -127,6 +127,7 @@ external_network:
 			--driver=bridge \
 			--gateway=203.0.113.1 \
 			--subnet=203.0.113.0/24 \
+			--ip-range=203.0.113.0/26 \
 			--opt "com.docker.network.driver.mtu=9000" \
 			--opt "com.docker.network.bridge.name=mini_lab_ext" \
 			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
@@ -276,7 +277,7 @@ ssh-machine:
 .PHONY: test-connectivity-to-external-service
 test-connectivity-to-external-service:
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
-		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://203.0.113.10" > /dev/null 2>&1; then \
+		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://203.0.113.100" > /dev/null 2>&1; then \
 			echo "Connected successfully"; \
 			exit 0; \
 		else \
@@ -295,8 +296,8 @@ test-connectivity-to-external-service:
 
 .PHONY: dev-env
 dev-env:
-	@echo "export METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal"
-	@echo "export METALCTL_HMAC=metal-admin"
+	@echo "export METALCTL_API_URL=${METALCTL_API_URL}"
+	@echo "export METALCTL_HMAC=${METALCTL_HMAC}"
 	@echo "export KUBECONFIG=$(KUBECONFIG)"
 
 ## Gardener integration

docs/network.svg

@@ -1,7 +1,8 @@
 {
   "CN": "metal-api",
   "hosts": [
-    "172.17.0.1"
+    "172.17.0.1",
+    "203.0.113.1"
   ],
   "key": {
     "algo": "rsa",

files/certs/grpc/server.json

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -o errexit -o xtrace
 
-ip addr add 203.0.113.10/24 dev mini_lab_ext
+ip addr add 203.0.113.100/24 dev mini_lab_ext
 ip route add 203.0.113.128/25 via 203.0.113.128 dev mini_lab_ext

files/external_service/network.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 import fcntl
 import glob
+import ipaddress
 import json
 import logging
 import os
@@ -216,6 +217,8 @@ def wait_until_all_interfaces_are_connected(interfaces: int) -> None:
         time.sleep(1)
 
 
+# This function works only for IPv4 interfaces.
+# See: man 7 netdevice
 def get_ip_address(iface: str) -> str:
     # Source: https://bit.ly/3dROGBN
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -226,6 +229,20 @@ def get_ip_address(iface: str) -> str:
     )[20:24])
 
 
+# This function works only for IPv4 interfaces
+# See: man 7 netdevice
+def get_netmask(iface: str) -> str:
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    netmask = socket.inet_ntoa(fcntl.ioctl(
+        s.fileno(),
+        0x891b,  # SIOCGIFNETMASK
+        struct.pack('256s', iface.encode('utf-8'))
+    )[20:24])
+    return str(ipaddress.ip_network(f"0.0.0.0/{netmask}").prefixlen)
+
+
+# This function works only for IPv4 interfaces
+# Set: man 7 netdevice
 def get_mac_address(iface: str) -> str:
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     mac = fcntl.ioctl(
@@ -236,6 +253,7 @@ def get_mac_address(iface: str) -> str:
     return ':'.join('%02x' % b for b in mac)
 
 
+# This function works only for IPv4 interfaces
 def get_default_gateway() -> str:
     # Source: https://splunktool.com/python-get-default-gateway-for-a-local-interfaceip-address-in-linux
     with open("/proc/net/route") as fh:
@@ -284,6 +302,7 @@ def parse_port_config() -> dict[str, dict]:
 
 
 def create_config_db(hwsku: str) -> dict:
+    mgmt_interface_cidr = get_ip_address("eth0") + "/" + get_netmask("eth0")
     return {
         'AUTO_TECHSUPPORT': {
             'GLOBAL': {
@@ -315,7 +334,7 @@ def create_config_db(hwsku: str) -> dict:
             }
         },
         'MGMT_INTERFACE': {
-            f'eth0|{get_ip_address("eth0")}/16': {
+            f'eth0|{mgmt_interface_cidr}': {
                 'gwaddr': get_default_gateway(),
             }
         },

images/sonic/launch.py

@@ -1,6 +1,6 @@
 ---
 metal_set_resource_limits: no
-metal_check_api_health_endpoint: http://api.172.17.0.1.nip.io:8080/metal/v1/health
+metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health
 
 # metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane
 

…ies/group_vars/control-plane/common.yaml …tories/group_vars/all/control_plane.yamlinventories/group_vars/control-plane/common.yaml renamed to inventories/group_vars/all/control_plane.yaml inventories/group_vars/control-plane/common.yaml renamed to inventories/group_vars/all/control_plane.yaml

@@ -3,7 +3,7 @@ metal_partition_timezone: Europe/Berlin
 metal_partition_id: mini-lab
 
 metal_partition_metal_api_protocol: http
-metal_partition_metal_api_addr: api.172.17.0.1.nip.io
+metal_partition_metal_api_addr: api.{{ metal_control_plane_ingress_dns }}
 metal_partition_metal_api_port: 8080
 metal_partition_metal_api_basepath: /metal/
 metal_partition_metal_api_hmac_edit_key: metal-edit

inventories/group_vars/control-plane/metal.yml

@@ -15,10 +15,7 @@ sonic_mgmtif_gateway: "172.17.0.1"
 
 sonic_mgmt_vrf: false
 
-sonic_nameservers:
-  - "172.17.0.1"
-  - "1.1.1.1"
-  - "1.0.0.1"
+sonic_nameservers: "{{ router_nameservers }}"
 
 sonic_ip_masquerade: true
 

inventories/group_vars/partition/common.yaml

@@ -3,15 +3,10 @@ name: mini-lab
 prefix: ""
 
 mgmt:
-  network: bridge
+  network: mini_lab_ext
 
 topology:
   nodes:
-    metal-control-plane-control-plane:
-      kind: ext-container
-      exec:
-        - ip addr add 203.0.113.100/24 dev mini_lab_ext
-        - ip route add 203.0.113.128/25 via 203.0.113.128 dev mini_lab_ext
     exit:
       kind: linux
       image: quay.io/frrouting/frr:10.0.1
@@ -51,8 +46,6 @@ topology:
         QEMU_MACHINE_CPU_CORES: 2
         QEMU_MACHINE_DISK_SIZE: 20G
   links:
-    - endpoints: ["metal-control-plane-control-plane:mini_lab_ext", "mini_lab_ext:kind"]
-      mtu: 9000
     - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"]
       mtu: 9000
     - endpoints: ["leaf01:Ethernet0", "vms:lan0"]