Merge pull request #5341 from ormergi/bgp-podman-support

Enable developemet around BGP using podman

Author: trozet

contrib/kind-common

@@ -174,16 +174,16 @@ EOF
   # Override GOBIN until https://github.com/metallb/metallb/issues/2218 is fixed.
   GOBIN="" inv dev-env -n ovn -b frr -p bgp -i "${ip_family}"
 
-  docker network rm -f clientnet
-  docker network create --subnet="${METALLB_CLIENT_NET_SUBNET_IPV4}" ${ipv6_network} --driver bridge clientnet
-  docker network connect clientnet frr
+  $OCI_BIN network rm -f clientnet
+  $OCI_BIN network create --subnet="${METALLB_CLIENT_NET_SUBNET_IPV4}" ${ipv6_network} --driver bridge clientnet
+  $OCI_BIN network connect clientnet frr
   if  [ "$PLATFORM_IPV6_SUPPORT" == true ]; then
     # Enable IPv6 forwarding in FRR
-    docker exec frr sysctl -w net.ipv6.conf.all.forwarding=1
+    $OCI_BIN exec frr sysctl -w net.ipv6.conf.all.forwarding=1
   fi
   # Note: this image let's us use it also for creating load balancer backends that can send big packets
-  docker rm -f lbclient
-  docker run  --cap-add NET_ADMIN --user 0  -d --network clientnet  --rm  --name lbclient  quay.io/itssurya/dev-images:metallb-lbservice
+  $OCI_BIN rm -f lbclient
+  $OCI_BIN run  --cap-add NET_ADMIN --user 0  -d --network clientnet  --rm  --name lbclient  quay.io/itssurya/dev-images:metallb-lbservice
   popd
   delete_metallb_dir
 
@@ -197,18 +197,18 @@ EOF
     kubectl label node "$n" node.kubernetes.io/exclude-from-external-load-balancers-
   done
 
-  kind_network_v4=$(docker inspect -f '{{index .NetworkSettings.Networks "kind" "IPAddress"}}' frr)
+  kind_network_v4=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.kind.IPAddress}}' frr)
   echo "FRR kind network IPv4: ${kind_network_v4}"
-  kind_network_v6=$(docker inspect -f '{{index .NetworkSettings.Networks "kind" "GlobalIPv6Address"}}' frr)
+  kind_network_v6=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' frr)
   echo "FRR kind network IPv6: ${kind_network_v6}"
   local client_network_v4 client_network_v6
-  client_network_v4=$(docker inspect -f '{{index .NetworkSettings.Networks "clientnet" "IPAddress"}}' frr)
+  client_network_v4=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.clientnet.IPAddress}}' frr)
   echo "FRR client network IPv4: ${client_network_v4}"
-  client_network_v6=$(docker inspect -f '{{index .NetworkSettings.Networks "clientnet" "GlobalIPv6Address"}}' frr)
+  client_network_v6=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.clientnet.GlobalIPv6Address}}' frr)
   echo "FRR client network IPv6: ${client_network_v6}"
 
   local client_subnets
-  client_subnets=$(docker network inspect clientnet -f '{{range .IPAM.Config}}{{.Subnet}}#{{end}}')
+  client_subnets=$($OCI_BIN network inspect clientnet -f '{{range .IPAM.Config}}{{.Subnet}}#{{end}}')
   echo "${client_subnets}"
   local client_subnets_v4 client_subnets_v6
   client_subnets_v4=$(echo "${client_subnets}" | cut -d '#' -f 1)
@@ -219,21 +219,21 @@ EOF
   KIND_NODES=$(kind_get_nodes)
   for n in ${KIND_NODES}; do
     if [ "$PLATFORM_IPV4_SUPPORT" == true ]; then
-        docker exec "${n}" ip route add "${client_subnets_v4}" via "${kind_network_v4}"
+        $OCI_BIN exec "${n}" ip route add "${client_subnets_v4}" via "${kind_network_v4}"
     fi
     if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then
-        docker exec "${n}" ip -6 route add "${client_subnets_v6}" via "${kind_network_v6}"
+        $OCI_BIN exec "${n}" ip -6 route add "${client_subnets_v6}" via "${kind_network_v6}"
     fi
   done
 
   # for now, we only run one test with metalLB load balancer for which this
   # one svcVIP (192.168.10.0/fc00:f853:ccd:e799::) is more than enough since at a time we will only
   # have one load balancer service
   if [ "$PLATFORM_IPV4_SUPPORT" == true ]; then
-    docker exec lbclient ip route add 192.168.10.0 via "${client_network_v4}" dev eth0
+    $OCI_BIN exec lbclient ip route add 192.168.10.0 via "${client_network_v4}" dev eth0
   fi
   if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then
-    docker exec lbclient ip -6 route add fc00:f853:ccd:e799:: via "${client_network_v6}" dev eth0
+    $OCI_BIN exec lbclient ip -6 route add fc00:f853:ccd:e799:: via "${client_network_v6}" dev eth0
   fi
   sleep 30
 }
@@ -254,14 +254,14 @@ install_plugins() {
 }
 
 destroy_metallb() {
-  if docker ps --format '{{.Names}}' | grep -Eq '^lbclient$'; then
-      docker stop lbclient
+  if $OCI_BIN ps --format '{{.Names}}' | grep -Eq '^lbclient$'; then
+      $OCI_BIN stop lbclient
   fi
-  if docker ps --format '{{.Names}}' | grep -Eq '^frr$'; then
-      docker stop frr
+  if $OCI_BIN ps --format '{{.Names}}' | grep -Eq '^frr$'; then
+      $OCI_BIN stop frr
   fi
-  if docker network ls --format '{{.Name}}' | grep -q '^clientnet$'; then
-      docker network rm clientnet
+  if $OCI_BIN network ls --format '{{.Name}}' | grep -q '^clientnet$'; then
+      $OCI_BIN network rm clientnet
   fi
   delete_metallb_dir
 }
@@ -708,7 +708,7 @@ deploy_frr_external_container() {
   popd || exit 1
   if  [ "$PLATFORM_IPV6_SUPPORT" == true ]; then
     # Enable IPv6 forwarding in FRR
-    docker exec frr sysctl -w net.ipv6.conf.all.forwarding=1
+    $OCI_BIN exec frr sysctl -w net.ipv6.conf.all.forwarding=1
   fi
 }
 
@@ -735,40 +735,40 @@ deploy_bgp_external_server() {
     ip_family="ipv4"
     ipv6_network=""
   fi
-  docker rm -f bgpserver
-  docker network rm -f bgpnet
-  docker network create --subnet="${BGP_SERVER_NET_SUBNET_IPV4}" ${ipv6_network} --driver bridge bgpnet
-  docker network connect bgpnet frr
-  docker run  --cap-add NET_ADMIN --user 0  -d --network bgpnet  --rm  --name bgpserver -p 8080:8080  registry.k8s.io/e2e-test-images/agnhost:2.45 netexec
+  $OCI_BIN rm -f bgpserver
+  $OCI_BIN network rm -f bgpnet
+  $OCI_BIN network create --subnet="${BGP_SERVER_NET_SUBNET_IPV4}" ${ipv6_network} --driver bridge bgpnet
+  $OCI_BIN network connect bgpnet frr
+  $OCI_BIN run  --cap-add NET_ADMIN --user 0  -d --network bgpnet  --rm  --name bgpserver -p 8080:8080  registry.k8s.io/e2e-test-images/agnhost:2.45 netexec
   # let's make the bgp external server have its default route towards FRR router so that we don't need to add routes during tests back to the pods in the
   # cluster for return traffic
   local bgp_network_frr_v4 bgp_network_frr_v6
-  bgp_network_frr_v4=$($OCI_BIN inspect -f '{{index .NetworkSettings.Networks "bgpnet" "IPAddress"}}' frr)
+  bgp_network_frr_v4=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.bgpnet.IPAddress}}' frr)
   echo "FRR kind network IPv4: ${bgp_network_frr_v4}"
   $OCI_BIN exec bgpserver ip route replace default via "$bgp_network_frr_v4"
   if  [ "$PLATFORM_IPV6_SUPPORT" == true ] ; then
-    bgp_network_frr_v6=$($OCI_BIN inspect -f '{{index .NetworkSettings.Networks "bgpnet" "GlobalIPv6Address"}}' frr)
+    bgp_network_frr_v6=$($OCI_BIN inspect -f '{{.NetworkSettings.Networks.bgpnet.GlobalIPv6Address}}' frr)
     echo "FRR kind network IPv6: ${bgp_network_frr_v6}"
     $OCI_BIN exec bgpserver ip -6 route replace default via "$bgp_network_frr_v6"
   fi
   # disable the default route to make sure the container only routes accross
   # directly connected or learnt networks (doing this at the very end since
   # docker changes the routing table when a new network is connected)
-  docker exec frr ip route delete default
-  docker exec frr ip route
-  docker exec frr ip -6 route delete default
-  docker exec frr ip -6 route
+  $OCI_BIN exec frr ip route delete default
+  $OCI_BIN exec frr ip route
+  $OCI_BIN exec frr ip -6 route delete default
+  $OCI_BIN exec frr ip -6 route
 }
 
 destroy_bgp() {
-  if docker ps --format '{{.Names}}' | grep -Eq '^bgpserver$'; then
-      docker stop bgpserver
+  if $OCI_BIN ps --format '{{.Names}}' | grep -Eq '^bgpserver$'; then
+      $OCI_BIN stop bgpserver
   fi
-  if docker ps --format '{{.Names}}' | grep -Eq '^frr$'; then
-      docker stop frr
+  if $OCI_BIN ps --format '{{.Names}}' | grep -Eq '^frr$'; then
+      $OCI_BIN stop frr
   fi
-  if docker network ls --format '{{.Name}}' | grep -q '^bgpnet$'; then
-      docker network rm bgpnet
+  if $OCI_BIN network ls --format '{{.Name}}' | grep -q '^bgpnet$'; then
+      $OCI_BIN network rm bgpnet
   fi
 }
 
@@ -807,7 +807,7 @@ install_ffr_k8s() {
 echo "Attempting to reach frr-k8s webhook"
 kind export kubeconfig --name ovn
 while true; do
-docker exec ovn-control-plane curl -ksS --connect-timeout 0.1 https://$(kubectl get svc -n frr-k8s-system frr-k8s-webhook-service -o jsonpath='{.spec.clusterIP}')
+$OCI_BIN exec ovn-control-plane curl -ksS --connect-timeout 0.1 https://$(kubectl get svc -n frr-k8s-system frr-k8s-webhook-service -o jsonpath='{.spec.clusterIP}')
 [ \$? -eq 0 ] && exit 0
 echo "Couldn't reach frr-k8s webhook, trying in 1s..."
 sleep 1s

contrib/kind.sh

@@ -42,6 +42,8 @@ function setup_kubectl_bin() {
 # The root cause is unknown, this also can not be reproduced in Ubuntu 20.04 or
 # with Fedora32 Cloud, but it does not happen if we clean first the ovn-kubernetes resources.
 delete() {
+  OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker}
+
   if [ "$KIND_INSTALL_METALLB" == true ]; then
     destroy_metallb
   fi

test/e2e/containerengine/container_engine.go

@@ -12,6 +12,16 @@ func (ce ContainerEngine) String() string {
 	return string(ce)
 }
 
+func (ce ContainerEngine) NetworkCIDRsFmt() string {
+	if ce == Podman {
+		return "{{json .Subnets }}"
+	}
+	if ce == Docker {
+		return "{{json .IPAM.Config }}"
+	}
+	return ""
+}
+
 const (
 	Docker ContainerEngine = "docker"
 	Podman ContainerEngine = "podman"

test/e2e/infraprovider/providers/kind/kind.go

@@ -414,7 +414,6 @@ func (c *contextKind) cleanUp() error {
 
 const (
 	nameFormat                     = "{{.Name}}"
-	inspectNetworkIPAMJSON         = "{{json .IPAM.Config }}"
 	inspectNetworkIPv4GWKeyStr     = "{{ .NetworkSettings.Networks.%s.Gateway }}"
 	inspectNetworkIPv4AddrKeyStr   = "{{ .NetworkSettings.Networks.%s.IPAddress }}"
 	inspectNetworkIPv4PrefixKeyStr = "{{ .NetworkSettings.Networks.%s.IPPrefixLen }}"
@@ -437,7 +436,7 @@ func isNetworkAttachedToContainer(networkName, containerName string) bool {
 
 func doesContainerNameExist(name string) bool {
 	// check if it is present before retrieving logs
-	stdOut, err := exec.Command(containerengine.Get().String(), "ps", "-f", fmt.Sprintf("Name=^%s$", name), "-q").CombinedOutput()
+	stdOut, err := exec.Command(containerengine.Get().String(), "ps", "-f", fmt.Sprintf("name=^%s$", name), "-q").CombinedOutput()
 	if err != nil {
 		panic(fmt.Sprintf("failed to check if external container (%s) exists: %v (%s)", name, err, stdOut))
 	}
@@ -466,13 +465,16 @@ func getNetwork(networkName string) (containerEngineNetwork, error) {
 		return n, api.NotFound
 	}
 	configs := make([]containerEngineNetworkConfig, 0, 1)
-	dataBytes, err := exec.Command(containerengine.Get().String(), "network", "inspect", "-f", inspectNetworkIPAMJSON, networkName).CombinedOutput()
+
+	ce := containerengine.Get()
+	netConfFmt := ce.NetworkCIDRsFmt()
+	dataBytes, err := exec.Command(ce.String(), "network", "inspect", "-f", netConfFmt, networkName).CombinedOutput()
 	if err != nil {
 		return n, fmt.Errorf("failed to extract network %q data: %v", networkName, err)
 	}
 	dataBytes = []byte(strings.Trim(string(dataBytes), "\n"))
 	if err = json.Unmarshal(dataBytes, &configs); err != nil {
-		return n, fmt.Errorf("failed to unmarshall network %q configuration using network inspect -f %q: %v", networkName, inspectNetworkIPAMJSON, err)
+		return n, fmt.Errorf("failed to unmarshall network %q configuration using network inspect -f %q: %v", networkName, netConfFmt, err)
 	}
 	if len(configs) == 0 {
 		return n, fmt.Errorf("failed to find any IPAM configuration for network %s", networkName)