diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2f77350c..98d97c16 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -7,6 +7,7 @@ on: push: branches: - master + - fix-incoming-ipv6 env: REGISTRY: ghcr.io diff --git a/Makefile b/Makefile index fccd5256..0b9c6303 100644 --- a/Makefile +++ b/Makefile @@ -54,8 +54,8 @@ up: env control-plane-bake partition-bake # without restarting the metal-core # TODO: should be investigated and fixed if possible sleep 10 - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf01 -i files/ssh/id_rsa 'systemctl restart metal-core' - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf02 -i files/ssh/id_rsa 'systemctl restart metal-core' + ssh -F files/ssh/config leaf01 'systemctl restart metal-core' + ssh -F files/ssh/config leaf02 'systemctl restart metal-core' .PHONY: restart restart: down up @@ -103,7 +103,9 @@ external_network: --opt "com.docker.network.driver.mtu=9000" \ --opt "com.docker.network.bridge.name=mini_lab_ext" \ --opt "com.docker.network.bridge.enable_ip_masquerade=true" && \ - sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi + sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext && \ + sudo ip route add 2001:db8:1:1::/80 via 2001:db8:1::2 dev mini_lab_ext; \ + fi .PHONY: env env: @@ -130,32 +132,17 @@ cleanup-partition: _privatenet: env docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network -.PHONY: _public_ips -_public_ips: env - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name firewall | grep firewall || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.129 --name firewall - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name machine | grep machine || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.130 --name machine - .PHONY: machine -machine: _privatenet _public_ips - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') +machine: _privatenet + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') .PHONY: firewall -firewall: _privatenet _public_ips +firewall: _privatenet docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') -# IPv6 -.PHONY: _privatenet6 -_privatenet6: env - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network-6 | grep user-private-network-6 || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000000 --name user-private-network-6 --addressfamily ipv6 - -.PHONY: machine6 -machine6: _privatenet6 - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test6 --name test6 --hostname test6 --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network-6 -o template --template '{{ .id }}') - -.PHONY: firewall6 -firewall6: _ips _privatenet6 - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --networks internet-ipv6-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network-6 -o template --template '{{ .id }}') - +.PHONY: public-ip +public-ip: + @docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name test --network internet-mini-lab -o template --template "{{ .ipaddress }}" .PHONY: ls ls: env @@ -178,11 +165,11 @@ docker-leaf02: .PHONY: ssh-leaf01 ssh-leaf01: - ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa root@leaf01 + ssh -F files/ssh/config leaf01 .PHONY: ssh-leaf02 ssh-leaf02: - ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa root@leaf02 + ssh -F files/ssh/config leaf02 ## MACHINE MANAGEMENT ## @@ -202,10 +189,6 @@ password-machine01: password-machine02: @$(MAKE) --no-print-directory _password MACHINE_UUID=2294c949-88f6-5390-8154-fa53d93a3313 -.PHONY: password-machine03 -password-machine03: - @$(MAKE) --no-print-directory _password MACHINE_UUID=2a92f14d-d3b1-4d46-b813-5d058103743e - .PHONY: _free-machine _free-machine: env docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine rm $(MACHINE_UUID) @@ -220,10 +203,6 @@ free-machine01: free-machine02: @$(MAKE) --no-print-directory _free-machine MACHINE_NAME=machine02 MACHINE_UUID=2294c949-88f6-5390-8154-fa53d93a3313 -.PHONY: free-machine03 -free-machine03: - @$(MAKE) --no-print-directory _free-machine MACHINE_NAME=machine03 MACHINE_UUID=2a92f14d-d3b1-4d46-b813-5d058103743e - .PHONY: _console-machine _console-machine: @echo "exit console with CTRL+5 and then quit telnet through q + ENTER" @@ -237,10 +216,6 @@ console-machine01: console-machine02: @$(MAKE) --no-print-directory _console-machine CONSOLE_PORT=4001 -.PHONY: console-machine03 -console-machine03: - @$(MAKE) --no-print-directory _console-machine CONSOLE_PORT=4002 - ## SSH TARGETS FOR MACHINES ## # Python code could be replaced by jq, but it is not preinstalled on Cumulus .PHONY: ssh-firewall diff --git a/README.md b/README.md index 94f5c627..1f4ce74f 100644 --- a/README.md +++ b/README.md @@ -195,15 +195,14 @@ docker compose run --rm metalctl machine rm e0ab02d2-27cd-5a5e-8efc-080ba80cf258 ## Flavors -There's few versions of mini-lab environment that you can run. We call them flavors. There's 2 flavors at the moment: +There are two versions, or flavors, of the mini-lab environment which differ in regards to the NOS running on the leaves: -- `default` -- runs 2 machines. -- `cluster-api` -- runs 3 machines. Useful for testing Control plane and worker node deployment with [Cluster API provider](https://github.com/metal-stack/cluster-api-provider-metalstack). -- `sonic` -- use SONiC as network operating system for the leaves +- `cumulus` -- runs 2 Cumulus switches. +- `sonic` -- runs 2 SONiC switches In order to start specific flavor, you can define the flavor as follows: ```bash -export MINI_LAB_FLAVOR=cluster-api +export MINI_LAB_FLAVOR=sonic make ``` diff --git a/files/ignition.json b/files/ignition.json index 7c2eefe3..354172c9 100644 --- a/files/ignition.json +++ b/files/ignition.json @@ -14,7 +14,7 @@ "path": "/etc/hosts.allow", "append": true, "contents": { - "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0AALL%3A%20203.0.113.1%0D%0A", + "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0AALL%3A%20203.0.113.1%0D%0AALL%3A%202001%3Adb8%3A1%3A%3A1%0D%0A", "verification": {} }, "mode": 644 diff --git a/files/rules.yaml b/files/rules.yaml index 4b97f8d4..89ed3432 100644 --- a/files/rules.yaml +++ b/files/rules.yaml @@ -40,3 +40,9 @@ ingress: - 203.0.113.1/25 to: - 203.0.113.128/25 + - comment: allow incoming ssh ipv6 + ports: + - 22 + protocol: TCP + to: + - 2001:db8:1:1::1/80 diff --git a/files/ssh/config b/files/ssh/config index 3bfcfc11..265faa6e 100644 --- a/files/ssh/config +++ b/files/ssh/config @@ -1,17 +1,14 @@ -Host leaf01 - HostName leaf01 +Host leaf* User root - IdentityFile files/ssh/id_rsa - PasswordAuthentication no - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - PubkeyAcceptedKeyTypes +ssh-rsa -Host * !leaf01 +Host * User metal IdentityFile files/ssh/id_rsa PasswordAuthentication no StrictHostKeyChecking no UserKnownHostsFile /dev/null + PubkeyAcceptedKeyTypes +ssh-rsa + +Host fe80::* # bash could be replaced with ncat, but it is not preinstalled on Cumulus ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"' diff --git a/images/sonic/config_db.json b/images/sonic/config_db.json index 02977d12..b34fca11 100644 --- a/images/sonic/config_db.json +++ b/images/sonic/config_db.json @@ -55,21 +55,13 @@ "admin_status": "up", "mtu": "9100" }, - "Ethernet8": { + "Ethernet120": { "lanes": "33,34,35,36", "alias": "fortyGigE0/8", "index": "2", "speed": "40000", "admin_status": "up", "mtu": "9100" - }, - "Ethernet12": { - "lanes": "37,38,39,40", - "alias": "fortyGigE0/12", - "index": "3", - "speed": "40000", - "admin_status": "up", - "mtu": "9100" } }, "VERSIONS": { @@ -77,4 +69,4 @@ "VERSION": "version_202311_03" } } -} +} \ No newline at end of file diff --git a/inventories/group_vars/all/images.yaml b/inventories/group_vars/all/images.yaml index 384fed0a..f0978e82 100644 --- a/inventories/group_vars/all/images.yaml +++ b/inventories/group_vars/all/images.yaml @@ -7,6 +7,7 @@ metal_stack_release_version: develop # metal_hammer_image_url: https://images.metal-stack.io/metal-hammer/pull-requests//metal-hammer-initrd.img.lz4 # metal_api_image_name: +metal_python_version: v0.39.2 metal_api_image_tag: dualstack-support # metal_metalctl_image_name: metal_metalctl_image_tag: 256-dualstack-support diff --git a/inventories/group_vars/sonic/main.yaml b/inventories/group_vars/sonic/main.yaml index 557f9bd3..fddd86ab 100644 --- a/inventories/group_vars/sonic/main.yaml +++ b/inventories/group_vars/sonic/main.yaml @@ -3,7 +3,7 @@ dhcp_listening_interfaces: - Vlan4000 metal_core_spine_uplinks: - - Ethernet0 + - Ethernet120 sonic_docker_routing_config_mode: split-unified sonic_frr_mgmt_framework_config: false diff --git a/mini-lab.cumulus.yaml b/mini-lab.cumulus.yaml index 30e43971..65aaf2ca 100644 --- a/mini-lab.cumulus.yaml +++ b/mini-lab.cumulus.yaml @@ -53,7 +53,5 @@ topology: - endpoints: ["leaf02:swp1", "vms:lan1"] - endpoints: ["leaf01:swp2", "vms:lan2"] - endpoints: ["leaf02:swp2", "vms:lan3"] - - endpoints: ["leaf01:swp3", "vms:lan4"] - - endpoints: ["leaf02:swp3", "vms:lan5"] - endpoints: ["leaf01:swp31", "inet:eth1"] - endpoints: ["leaf02:swp31", "inet:eth2"] diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 73c6863e..b0ac58e2 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -55,11 +55,9 @@ topology: - endpoints: ["inet:ext", "mini_lab_ext:inet"] mtu: 9000 - endpoints: ["www:ext", "mini_lab_ext:www"] - - endpoints: ["leaf01:eth1", "inet:eth1"] - - endpoints: ["leaf02:eth1", "inet:eth2"] - - endpoints: ["leaf01:eth2", "vms:lan0"] - - endpoints: ["leaf02:eth2", "vms:lan1"] - - endpoints: ["leaf01:eth3", "vms:lan2"] - - endpoints: ["leaf02:eth3", "vms:lan3"] - - endpoints: ["leaf01:eth4", "vms:lan4"] - - endpoints: ["leaf02:eth4", "vms:lan5"] + - endpoints: ["leaf01:eth1", "vms:lan0"] + - endpoints: ["leaf02:eth1", "vms:lan1"] + - endpoints: ["leaf01:eth2", "vms:lan2"] + - endpoints: ["leaf02:eth2", "vms:lan3"] + - endpoints: ["leaf01:eth3", "inet:eth1"] + - endpoints: ["leaf02:eth3", "inet:eth2"] diff --git a/scripts/manage_vms.py b/scripts/manage_vms.py index 2f2008cf..0b015ec4 100755 --- a/scripts/manage_vms.py +++ b/scripts/manage_vms.py @@ -25,15 +25,6 @@ "lan_indices": [2, 3], "serial-port": 4001, }, - "machine03": { - "name": "machine03", - "uuid": "2a92f14d-d3b1-4d46-b813-5d058103743e", - "disk-path": "/machine03.img", - "disk-size": "5G", - "memory": "2G", - "lan_indices": [4, 5], - "serial-port": 4002, - }, } diff --git a/test/ci-cleanup.sh b/test/ci-cleanup.sh index a2160898..52a99370 100755 --- a/test/ci-cleanup.sh +++ b/test/ci-cleanup.sh @@ -15,3 +15,11 @@ if [ ! -z "$running_containers" ]; then fi make cleanup + +echo "Remove containers from previous runs" + +previous_mini_lab_containers=$(docker container list --all --filter label=containerlab=mini-lab --quiet) + +if [ ! -z "$previous_mini_lab_containers" ]; then + docker container rm --force $(docker container list --all --filter label=containerlab=mini-lab --quiet) +fi diff --git a/test/integration.sh b/test/integration.sh index 91964db7..85ff881c 100755 --- a/test/integration.sh +++ b/test/integration.sh @@ -49,6 +49,6 @@ echo "Test connectivity to outside ipv6" make connect-to-www-ipv6 echo "Test connectivity from outside" -ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.130 -C exit +ssh -F files/ssh/config metal@$(make public-ip) -C exit echo "Successfully started mini-lab"