Introduce new mini-lab flavor for Enterprise SONiC and fix capi-lab (#272)

This adds yet another flavor (`dell_sonic`) to the mini-lab for Enterprise SONiC (Dell, Broadcom).
Version 4.4.3. was tested initially. By default we test against version 4.5.1.

* Add flavor for dell sonic

* DROP ME: use image with installed sshpass

* Fix README

* Use known working metal-core for Enterprise SONiC

* Fix metal_core_spine_uplinks

* The kind dell_sonic only supports 10 interfaces

* Fix Makefile targets for dell_sonic

* Set hello timer for LLDP

* Activate IP MASQUERADE and IPv4 forwarding on eth0

* Reformat task names

* Adjust README.md

* trying to fix issues with docker pull and lldp ports config

* login to r.metal-stack.io

* use proper image

* pull dell_sonic image within CI

* add a Makefile dev target to build vrnetlab/dell_sonic with a given sonic-vs.img

* use old metal-core version

* check for queued routes

* use dell_sonic for capms flavor

* capi needs more disk space + fix lldp for Ethernet2 to also get machine03

* Add fourth machine

* adapt README.md

* ipv6 lla at vlan interfaces + longer wait time for metal-core

* wait

* fix

* use released version of metal-core

* use version 4.5.1

* use common deployment image

* enterprise sonic 4.4.3 and 4.5.1

* use metal-core@fix-ethernet-packet-overflow

* use [email protected]

* use metal-core from release_version develop

* remove unnecessary files

* use deployment_base_image from release vector

* use proper database version 4.5.3 for Enterprise SONiC 4.5.1

---------

Co-authored-by: Robert Volkmann <[email protected]>
Co-authored-by: Ilja Rotar <[email protected]>
Co-authored-by: Simon Mayer <[email protected]>

Author: 4 people

.github/workflows/integration.yaml

@@ -97,6 +97,7 @@ jobs:
         flavors:
           - name: sonic
           - name: gardener
+          - name: dell_sonic
 
     steps:
     - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211
@@ -111,6 +112,13 @@ jobs:
         # set if required:
         # DESIRED_VERSION: v0.59.0
 
+    - name: Log in to the container registry
+      uses: docker/login-action@v3
+      with:
+        registry: r.metal-stack.io
+        username: ${{ secrets.R_METALSTACK_IO_READ_USER }}
+        password: ${{ secrets.R_METALSTACK_IO_READ_PASSWORD }}
+
     - name: Checkout
       uses: actions/checkout@v4
 

.gitignore

@@ -17,3 +17,4 @@ files/certs/*.pem
 files/certs/**/*.pem
 files/certs/**/*.crt
 .vscode
+vrnetlab

Makefile

@@ -22,15 +22,20 @@ ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),)
 MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
+MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
 
 MACHINE_OS=debian-12.0
 MAX_RETRIES := 30
 
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
+LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
+MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
 else ifeq ($(MINI_LAB_FLAVOR),capms)
 LAB_TOPOLOGY=mini-lab.capms.yaml
+MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
 else ifeq ($(MINI_LAB_FLAVOR),gardener)
 GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
@@ -59,13 +64,10 @@ up: env gen-certs control-plane-bake partition-bake
 # for some reason an allocated machine will not be able to phone home
 # without restarting the metal-core
 # TODO: should be investigated and fixed if possible
+# check that underlay gets working
 	sleep 10
 	ssh -F files/ssh/config leaf01 'systemctl restart metal-core'
 	ssh -F files/ssh/config leaf02 'systemctl restart metal-core'
-# TODO: for community SONiC versions > 202311 a bgp restart is needed in the virtual environment
-	sleep 15
-	ssh -F files/ssh/config leaf01 'systemctl restart bgp'
-	ssh -F files/ssh/config leaf02 'systemctl restart bgp'
 
 .PHONY: restart
 restart: down up
@@ -113,7 +115,12 @@ partition: partition-bake
 .PHONY: partition-bake
 partition-bake: external_network
 	docker pull $(MINI_LAB_VM_IMAGE)
+ifeq ($(CI),true)
+	docker pull $(MINI_LAB_SONIC_IMAGE)
+endif
+ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms),$(MINI_LAB_FLAVOR))
 	docker pull $(MINI_LAB_SONIC_IMAGE)
+endif
 	@if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \
 		sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
 		./scripts/deactivate_offloading.sh; fi
@@ -152,6 +159,7 @@ cleanup-control-plane:
 .PHONY: cleanup-partition
 cleanup-partition:
 	mkdir -p clab-mini-lab
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
 	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.capms.yaml
 	docker network rm --force mini_lab_ext
@@ -249,6 +257,10 @@ power-on-machine02:
 power-on-machine03:
 	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=on
 
+.PHONY: power-on-machine04
+power-on-machine04:
+	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=on
+
 .PHONY: power-reset-machine01
 power-reset-machine01:
 	@$(MAKE) --no-print-directory _ipmi_power VM=machine01 COMMAND=reset
@@ -261,6 +273,10 @@ power-reset-machine02:
 power-reset-machine03:
 	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=reset
 
+.PHONY: power-reset-machine04
+power-reset-machine04:
+	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=reset
+
 .PHONY: power-off-machine01
 power-off-machine01:
 	@$(MAKE) --no-print-directory _ipmi_power VM=machine01 COMMAND=off
@@ -273,6 +289,10 @@ power-off-machine02:
 power-off-machine03:
 	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=off
 
+.PHONY: power-off-machine04
+power-off-machine04:
+	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=off
+
 .PHONY: _console
 _console:
 	docker exec --interactive --tty $(VM) ipmitool -C 3 -I lanplus -U ADMIN -P ADMIN -H 127.0.0.1 sol activate
@@ -289,6 +309,10 @@ console-machine02:
 console-machine03:
 	@$(MAKE) --no-print-directory _console VM=machine03
 
+.PHONY: console-machine04
+console-machine04:
+	@$(MAKE) --no-print-directory _console VM=machine04
+
 .PHONY: _password
 _password: env
 	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine consolepassword $(MACHINE_UUID)
@@ -301,6 +325,14 @@ password-machine01:
 password-machine02:
 	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine02 MACHINE_UUID=00000000-0000-0000-0000-000000000002
 
+.PHONY: password-machine03
+password-machine03:
+	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine03 MACHINE_UUID=00000000-0000-0000-0000-000000000003
+
+.PHONY: password-machine04
+password-machine04:
+	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine04 MACHINE_UUID=00000000-0000-0000-0000-000000000004
+
 .PHONY: password-machine0%
 password-machine0%:
 	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine0$* MACHINE_UUID=00000000-0000-0000-0000-00000000000$*
@@ -371,6 +403,17 @@ dev-env:
 	@echo "export METALCTL_HMAC=${METALCTL_HMAC}"
 	@echo "export KUBECONFIG=$(KUBECONFIG)"
 
+build-dell-sonic:
+	if [ ! -f "sonic-vs.img" ]; then \
+	    @echo "sonic-vs.img is expected in this directory"; exit; fi
+
+	@git clone https://github.com/srl-labs/vrnetlab.git
+	@cd vrnetlab && git checkout e41f48bc5cae777b56b71b67e3c5642fdbd8f315
+	@cp ./sonic-vs.img vrnetlab/dell/dell_sonic/dell-sonic-$(MINI_LAB_DELL_SONIC_VERSION).qcow2
+	@cd vrnetlab/dell/dell_sonic && make
+	docker tag vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION) r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
+	@rm -rf ./vrnetlab
+
 ## Gardener integration
 
 .PHONY: fetch-virtual-kubeconfig

README.md

@@ -190,11 +190,12 @@ make power-<on,reset,off>-<machine name>
 
 ## Flavors
 
-There are two versions, or flavors, of the mini-lab environment which differ in regards to the NOS running on the leaves:
+There are four flavors of the mini-lab environment:
 
-- `sonic`: runs 2 SONiC switches
-- `capms`: runs the SONiC flavor but with three instead of two machines (this is used for  [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall)
-- `gardener`: installs the [Gardener](https://gardener.cloud) in the mini-lab
+- `sonic`: runs two Community SONiC switches
+- `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic)
+- `capms`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall)
+- `gardener`: runs the `sonic` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab
 
 In order to start specific flavor, you can define the flavor as follows:
 

deploy_partition.yaml

@@ -1,6 +1,6 @@
 ---
-- name: Configure leaves
-  hosts: leaves
+- name: Configure leaves (Community SONiC)
+  hosts: leaves:!dell_sonic
   any_errors_fatal: true
   gather_facts: false
   pre_tasks:
@@ -16,8 +16,44 @@
     - name: sonic
       tags: sonic
 
-- name: Deploy dhcp server and pixiecore on leaf01
-  hosts: leaf01
+- name: Configure leaves (Enterprise SONiC)
+  hosts: dell_sonic
+  any_errors_fatal: true
+  become: true
+  tasks:
+    - name: Check for bgpd.conf presence
+      ansible.builtin.stat:
+        path: /etc/sonic/frr/bgpd.conf
+      register: bgpd_conf_stat
+
+    - name: Restart bgp service when non-split docker_routing_config_mode is still active
+      ansible.builtin.service:
+        name: bgp
+        state: restarted
+      when: bgpd_conf_stat.stat.exists
+
+    - name: Ensure root has authorized_key
+      ansible.builtin.authorized_key:
+        user: root
+        state: present
+        key: "{{ lookup('file', 'ssh/id_rsa.pub') }}"
+
+    - name: Activate IP MASQUERADE on eth0
+      ansible.builtin.iptables:
+        chain: POSTROUTING
+        jump: MASQUERADE
+        out_interface: eth0
+        table: nat
+
+    - name: Activate IPv4 forwarding on eth0
+      ansible.posix.sysctl:
+        name: net.ipv4.conf.eth0.forwarding
+        reload: no
+        sysctl_set: yes
+        value: "1"
+
+- name: Deploy dhcp server on leaf01 (Community SONiC)
+  hosts: leaf01:!dell_sonic
   pre_tasks:
     - name: Temporary workaround for EOL debian bullseye backports repository (using archive.debian.org)
       lineinfile:
@@ -29,12 +65,36 @@
       tags: always
     - name: metal-roles/partition/roles/dhcp
       tags: dhcp
+
+# FIXME: For some reason, the first docker pull always fails on dell_sonic but succeeds on second attempt.
+# Investigate the cause and remove this play
+- name: Intentionally fail on first docker pull
+  hosts: dell_sonic
+  tasks:
+    - community.docker.docker_container:
+        name: hello-world
+        image: library/hello-world:latest
+        cleanup: true
+        pull: true
+      failed_when: false
+
+- name: Deploy pixiecore on leaf01
+  hosts: leaf01
+  become: true
+  roles:
+    - name: ansible-common
+      tags: always
     - name: metal-roles/partition/roles/pixiecore
       tags: pixiecore
 
 - name: Deploy metal-core
   hosts: leaves
   any_errors_fatal: true
+  become: true
+  pre_tasks:
+  - name: Wait some time
+    pause:
+      seconds: 120
   roles:
     - name: ansible-common
       tags: always
@@ -58,9 +118,36 @@
     - name: Wait for switches to register
       command: echo
       changed_when: false
-      retries: 60
+      retries: 100
       delay: 3
       until:
         - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key) | length == 2
         - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[0]["last_sync"] != None
         - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[1]["last_sync"] != None
+
+- name: Wait for underlay (Community SONiC)
+  hosts: leaves:!dell_sonic
+  any_errors_fatal: true
+  gather_facts: false
+  pre_tasks:
+  - name: Wait until no route entries have "queued"
+    include_tasks: tasks/check_queued.yaml
+
+- name: Configure IPv6 and LLDP ports (Enterprise SONiC)
+  hosts: dell_sonic
+  any_errors_fatal: true
+  become: true
+  tasks:
+    - name: Enable IPv6 to also have LLA at VLAN interfaces
+      sysctl:
+        name: net.ipv6.conf.default.disable_ipv6
+        value: '0'
+        state: present
+        sysctl_file: /etc/sysctl.conf
+    - name: Configure LLDP port IDs and descriptions
+      ansible.builtin.command: "{{ item }}"
+      with_items:
+        - lldpcli configure ports Ethernet0 lldp portidsubtype local Eth1/1
+        - lldpcli configure ports Ethernet1 lldp portidsubtype local Eth1/2
+        - lldpcli configure ports Ethernet2 lldp portidsubtype local Eth1/3
+        - lldpcli configure ports Ethernet3 lldp portidsubtype local Eth1/4

files/kea.json

@@ -0,0 +1,53 @@
+{
+  "Dhcp4": {
+    "option-data": [
+      {
+        "name": "domain-name-servers",
+        "data": "1.1.1.1, 8.8.8.8",
+        "always-send": true
+      }
+    ],
+    "subnet4": [
+      {
+        "id": 1,
+        "subnet": "10.0.1.0/25",
+        "pools": [
+          {
+            "pool": "10.0.1.2 - 10.0.1.127"
+          }
+        ],
+        "option-data": [
+          {
+            "name": "routers",
+            "data": "10.0.1.1"
+          }
+        ]
+      }
+    ],
+    "interfaces-config": {
+      "interfaces": [
+        "lo"
+      ],
+      "dhcp-socket-type": "udp",
+      "outbound-interface": "use-routing"
+    },
+    "control-socket": {
+      "socket-type": "unix",
+      "socket-name": "/run/kea/control_socket_4"
+    },
+    "loggers": [
+      {
+        "name": "kea-dhcp4",
+        "output_options": [
+          {
+            "output": "stdout"
+          }
+        ],
+        "severity": "DEBUG"
+      }
+    ],
+    "lease-database": {
+      "type": "memfile"
+    }
+  }
+}