BGP, shard: don't use frr as the default gateway

This PR changes BGP regression to test things
using the actual default gateway that exists
instead of using the frr router as its default
gateway.

This does two things:
1) add masquerade routes to github runner for these lanes
2) add routes from github runner back to the nodes for reply traffic

since github runner will be the default gateway for the
cluster and pod traffic will leave the cluster unSNATed,
we have to have the infra setup have these two above
pieces for the tests to work properly

Signed-off-by: Surya Seetharaman <[email protected]>

Author: tssurya

.github/workflows/test.yml

@@ -414,8 +414,8 @@ jobs:
         include:
           - {"target": "shard-conformance", "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
-          - {"target": "shard-conformance", "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"}
-          - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"}
+          - {"target": "shard-conformance", "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"}
+          - {"target": "shard-conformance", "ha": "HA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled",          "dns-name-resolver": "enable-dns-name-resolver"}
@@ -518,6 +518,30 @@ jobs:
           echo OVN_TEST_EX_GW_NETWORK=kindexgw >> $GITHUB_ENV
           echo OVN_ENABLE_EX_GW_NETWORK_BRIDGE=true >> $GITHUB_ENV
         fi
+        if [[ "$JOB_NAME" == *"shard-conformance"* ]] && [ "$ADVERTISE_DEFAULT_NETWORK" == "true" ]; then
+          echo "ADVERTISE_DEFAULT_NETWORK=true" >> $GITHUB_ENV
+
+          # Use proper variable declaration with default values
+          NET_CIDR_IPV4=${NET_CIDR_IPV4:-10.244.0.0/16}
+          NET_CIDR_IPV6=${NET_CIDR_IPV6:-fd00:10:244::/48}
+
+          sudo ip a
+          sudo ip r
+
+          # Add masquerade rules for both IPv4 and IPv6 networks
+          echo "Adding masquerade rule for $NET_CIDR_IPV4"
+          sudo iptables -t nat -A POSTROUTING -s $NET_CIDR_IPV4 -o eth0 -j MASQUERADE
+
+          echo "Adding masquerade rule for $NET_CIDR_IPV6"
+          sudo ip6tables -t nat -A POSTROUTING -s $NET_CIDR_IPV6 -o eth0 -j MASQUERADE
+
+          # Verify the rules were added
+          echo "IPv4 POSTROUTING rules:"
+          sudo iptables -t nat -L POSTROUTING -v
+
+          echo "IPv6 POSTROUTING rules:"
+          sudo ip6tables -t nat -L POSTROUTING -v
+        fi
 
     - name: Disable ufw
       # For IPv6 and Dualstack, ufw (Uncomplicated Firewall) should be disabled.

contrib/kind-common

@@ -679,16 +679,6 @@ deploy_frr_external_container() {
   sed -i '/remote-as 64512/a \ neighbor {{ . }} route-reflector-client' frr/frr.conf.tmpl
   ./demo.sh
   popd || exit 1
-
-  # this container will act as the gateway for the cluster and will masquerade
-  # towards the external world
-  $OCI_BIN exec frr iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
-  # set default route
-  FRR_IP=$($OCI_BIN inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" frr)
-  KIND_NODES=$(kind_get_nodes)
-  for n in $KIND_NODES; do
-    $OCI_BIN exec "$n" ip route replace default via "$FRR_IP"
-  done
 }
 
 install_ffr_k8s() {
@@ -708,4 +698,40 @@ install_ffr_k8s() {
   popd || exit 1
 
   rm -rf "${FRR_TMP_DIR}"
+  # Add routes for pod networks dynamically into the github runner for return traffic to pass back
+  if [ -n "${JOB_NAME:-}" ] && [[ "$JOB_NAME" == *"shard-conformance"* ]] && [ "$ADVERTISE_DEFAULT_NETWORK" == "true" ]; then
+    echo "Adding routes for Kubernetes pod networks..."
+    NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}')
+    echo "Found nodes: $NODES"
+    for node in $NODES; do
+      # Get the addresses
+      node_ips=$(kubectl get node $node -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+      # Get subnet information
+      subnet_json=$(kubectl get node $node -o jsonpath='{.metadata.annotations.k8s\.ovn\.org/node-subnets}')
+      
+      if [ "$KIND_IPV4_SUPPORT" == true ]; then
+        # Extract IPv4 address (first address)
+        node_ipv4=$(echo "$node_ips" | awk '{print $1}')
+        ipv4_subnet=$(echo "$subnet_json" | jq -r '.default[0]')
+        
+        # Add IPv4 route
+        if [ -n "$ipv4_subnet" ] && [ -n "$node_ipv4" ]; then
+          echo "Adding IPv4 route for $node ($node_ipv4): $ipv4_subnet"
+          sudo ip route add $ipv4_subnet via $node_ipv4
+        fi
+      fi
+
+      # Add IPv6 route if enabled
+      if [ "$KIND_IPV6_SUPPORT" == true ]; then
+        # Extract IPv6 address (second address, if present)
+        node_ipv6=$(echo "$node_ips" | awk '{print $2}')
+        ipv6_subnet=$(echo "$subnet_json" | jq -r '.default[1] // empty')
+        
+        if [ -n "$ipv6_subnet" ] && [ -n "$node_ipv6" ]; then
+          echo "Adding IPv6 route for $node ($node_ipv6): $ipv6_subnet"
+          sudo ip -6 route add $ipv6_subnet via $node_ipv6
+        fi
+      fi
+    done
+  fi
 }