add ipv6 default route to dualstack windows nodes (#2508)

* add ipv6 default route to dualstack windows nodes

* fix comments

* add UT for adding ipv6 default route

* add UT for adding ipv6 default route

* fix linter issue

* fix comments

* fix comments

* fix an issue

* fix comments and UT

* fix the UT test

* fix the subnet family afINET

* fix the UT test

* fix an UT test

* fix Ramiro's comments

* fix latest comments

* fix a linter issue

* add context to failure of UT test

* Update network/network_windows.go

Co-authored-by: tamilmani1989 <[email protected]>
Signed-off-by: Paul Yu <[email protected]>

* add an UT to  cover failure case

* change the UT name

* add UT to mock powershell command

* fix UT

* fix linter issue

---------

Signed-off-by: Paul Yu <[email protected]>
Co-authored-by: tamilmani1989 <[email protected]>

Author: paulyufan2

network/network_windows.go

@@ -5,17 +5,18 @@ package network
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/Azure/azure-container-networking/network/hnswrapper"
 	"github.com/Azure/azure-container-networking/network/policy"
+	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Microsoft/hcsshim"
 	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
@@ -38,6 +39,10 @@ const (
 	routeCmd = "netsh interface ipv6 %s route \"%s\" \"%s\" \"%s\" store=persistent"
 	// add/delete ipv4 and ipv6 route rules to/from windows node
 	netRouteCmd = "netsh interface %s %s route \"%s\" \"%s\" \"%s\""
+	// Default IPv6 Route
+	defaultIPv6Route = "::/0"
+	// Default IPv6 nextHop
+	defaultIPv6NextHop = "fe80::1234:5678:9abc"
 )
 
 // Windows implementation of route.
@@ -318,6 +323,34 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern
 	return hcnNetwork, nil
 }
 
+func (nm *networkManager) addIPv6DefaultRoute() error {
+	// add ipv6 default route if it does not exist in dualstack overlay windows node from persistent store
+	// persistent store setting is only read during the adapter restarts or reboots to re-populate the active store
+
+	// get interface index to add ipv6 default route and only consider there is one vEthernet interface for now
+	getIpv6IfIndexCmd := `((Get-NetIPInterface | where InterfaceAlias -Like "vEthernet*").IfIndex)[0]`
+	ifIndex, err := nm.plClient.ExecutePowershellCommand(getIpv6IfIndexCmd)
+	if err != nil {
+		return errors.Wrap(err, "error while executing powershell command to get ipv6 Hyper-V interface")
+	}
+
+	getIPv6RoutePersistentCmd := fmt.Sprintf("Get-NetRoute -DestinationPrefix %s -PolicyStore Persistentstore", defaultIPv6Route)
+	if out, err := nm.plClient.ExecutePowershellCommand(getIPv6RoutePersistentCmd); err != nil {
+		logger.Info("ipv6 default route is not found from persistentstore, adding default ipv6 route to the windows node", zap.String("out", out), zap.Error(err))
+		// run powershell cmd to add ipv6 default route
+		// if there is an ipv6 default route in active store but not persistent store; to add ipv6 default route to persistent store
+		// need to remove ipv6 default route from active store and then use this command to add default route entry to both active and persistent store
+		addCmd := fmt.Sprintf("Remove-NetRoute -DestinationPrefix %s -InterfaceIndex %s -NextHop %s -confirm:$false;New-NetRoute -DestinationPrefix %s -InterfaceIndex %s -NextHop %s -confirm:$false",
+			defaultIPv6Route, ifIndex, defaultIPv6NextHop, defaultIPv6Route, ifIndex, defaultIPv6NextHop)
+
+		if _, err := nm.plClient.ExecutePowershellCommand(addCmd); err != nil {
+			return errors.Wrap(err, "Failed to add ipv6 default route to both persistent and active store")
+		}
+	}
+
+	return nil
+}
+
 // newNetworkImplHnsV2 creates a new container network for HNSv2.
 func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) {
 	hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf)
@@ -347,6 +380,17 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern
 		logger.Info("Network with name already exists", zap.String("name", hcnNetwork.Name))
 	}
 
+	// check if ipv6 default gateway route is missing before windows endpoint creation
+	for i := range nwInfo.Subnets {
+		if nwInfo.Subnets[i].Family == platform.AfINET6 {
+			if err = nm.addIPv6DefaultRoute(); err != nil {
+				// should not block network creation but remind user that it's failed to add ipv6 default route to windows node
+				logger.Error("failed to add missing ipv6 default route to windows node active/persistent store", zap.Error(err))
+			}
+			break
+		}
+	}
+
 	var vlanid int
 	opt, _ := nwInfo.Options[genericData].(map[string]interface{})
 	if opt != nil && opt[VlanIDKey] != nil {

network/network_windows_test.go

@@ -7,14 +7,24 @@
 package network
 
 import (
+	"errors"
 	"fmt"
+	"net"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/Azure/azure-container-networking/network/hnswrapper"
+	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Microsoft/hcsshim/hcn"
 )
 
+var (
+	errTestFailure     = errors.New("test failure")
+	failedCaseReturn   = "false"
+	succededCaseReturn = "true"
+)
+
 func TestNewAndDeleteNetworkImplHnsV2(t *testing.T) {
 	nm := &networkManager{
 		ExternalInterfaces: map[string]*externalInterface{},
@@ -227,3 +237,166 @@ func TestDeleteNetworkImplHnsV1WithTimeout(t *testing.T) {
 		t.Fatal("Failed to timeout HNS calls for deleting network")
 	}
 }
+
+func TestAddIPv6DefaultRoute(t *testing.T) {
+	_, ipnetv4, _ := net.ParseCIDR("10.240.0.0/12")
+	_, ipnetv6, _ := net.ParseCIDR("fc00::/64")
+
+	nm := &networkManager{
+		ExternalInterfaces: map[string]*externalInterface{},
+		plClient:           platform.NewMockExecClient(false),
+	}
+
+	networkSubnetInfo := []SubnetInfo{
+		{
+			Family:  platform.AfINET,
+			Gateway: net.ParseIP("10.240.0.1"),
+			Prefix:  *ipnetv4,
+		},
+		{
+			Family:  platform.AfINET6,
+			Gateway: net.ParseIP("fc00::1"),
+			Prefix:  *ipnetv6,
+		},
+	}
+
+	nwInfo := &NetworkInfo{
+		Id:           "d3f97a83-ba4c-45d5-ba88-dc56757ece28",
+		MasterIfName: "eth0",
+		Mode:         "bridge",
+		Subnets:      networkSubnetInfo,
+	}
+
+	extInterface := &externalInterface{
+		Name:    "eth0",
+		Subnets: []string{"subnet1", "subnet2"},
+	}
+
+	Hnsv2 = hnswrapper.NewHnsv2wrapperFake()
+
+	// check if network can be successfully created
+	_, err := nm.newNetworkImplHnsV2(nwInfo, extInterface)
+	if err != nil {
+		t.Fatalf("Failed to create network due to error:%+v", err)
+	}
+}
+
+func TestFailToAddIPv6DefaultRoute(t *testing.T) {
+	_, ipnetv4, _ := net.ParseCIDR("10.240.0.0/12")
+	_, ipnetv6, _ := net.ParseCIDR("fc00::/64")
+
+	nm := &networkManager{
+		ExternalInterfaces: map[string]*externalInterface{},
+		plClient:           platform.NewMockExecClient(true), // return mock exec error
+	}
+
+	networkSubnetInfo := []SubnetInfo{
+		{
+			Family:  platform.AfINET,
+			Gateway: net.ParseIP("10.240.0.1"),
+			Prefix:  *ipnetv4,
+		},
+		{
+			Family:  platform.AfINET6,
+			Gateway: net.ParseIP("fc00::1"),
+			Prefix:  *ipnetv6,
+		},
+	}
+
+	nwInfo := &NetworkInfo{
+		Id:           "d3f97a83-ba4c-45d5-ba88-dc56757ece28",
+		MasterIfName: "eth0",
+		Mode:         "bridge",
+		Subnets:      networkSubnetInfo,
+	}
+
+	extInterface := &externalInterface{
+		Name:    "eth0",
+		Subnets: []string{"subnet1", "subnet2"},
+	}
+
+	Hnsv2 = hnswrapper.NewHnsv2wrapperFake()
+
+	// check if network is failed to create
+	_, err := nm.newNetworkImplHnsV2(nwInfo, extInterface)
+	if err == nil {
+		t.Fatal("Network should not be created")
+	}
+}
+
+func TestAddIPv6DefaultRouteHappyPath(t *testing.T) {
+	mockExecClient := platform.NewMockExecClient(false)
+
+	nm := &networkManager{
+		plClient: mockExecClient,
+	}
+
+	// happy path
+	mockExecClient.SetPowershellCommandResponder(func(cmd string) (string, error) {
+		if strings.Contains(cmd, "Get-NetIPInterface") || strings.Contains(cmd, "Remove-NetRoute") {
+			return succededCaseReturn, nil
+		}
+
+		// fail secondary command execution and successfully execute remove-netRoute command
+		if strings.Contains(cmd, "Get-NetRoute") {
+			return failedCaseReturn, errTestFailure
+		}
+
+		return "", nil
+	})
+
+	err := nm.addIPv6DefaultRoute()
+	if err != nil {
+		t.Fatal("Failed to test happy path")
+	}
+}
+
+func TestAddIPv6DefaultRouteUnhappyPathGetNetInterface(t *testing.T) {
+	mockExecClient := platform.NewMockExecClient(false)
+
+	nm := &networkManager{
+		plClient: mockExecClient,
+	}
+
+	// failed to execute Get-NetIPInterface command to find interface index
+	mockExecClient.SetPowershellCommandResponder(func(cmd string) (string, error) {
+		if strings.Contains(cmd, "Get-NetIPInterface") {
+			return failedCaseReturn, errTestFailure
+		}
+		return "", nil
+	})
+
+	err := nm.addIPv6DefaultRoute()
+	if err == nil {
+		t.Fatal("Failed to test unhappy path with failing to execute get-netIPInterface command")
+	}
+}
+
+func TestAddIPv6DefaultRouteUnhappyPathAddRoute(t *testing.T) {
+	mockExecClient := platform.NewMockExecClient(false)
+
+	nm := &networkManager{
+		plClient: mockExecClient,
+	}
+
+	mockExecClient.SetPowershellCommandResponder(func(cmd string) (string, error) {
+		if strings.Contains(cmd, "Get-NetIPInterface") {
+			return succededCaseReturn, nil
+		}
+
+		// fail secondary command execution and failed to execute remove-netRoute command
+		if strings.Contains(cmd, "Get-NetRoute") {
+			return failedCaseReturn, errTestFailure
+		}
+
+		if strings.Contains(cmd, "Remove-NetRoute") {
+			return failedCaseReturn, errTestFailure
+		}
+		return "", nil
+	})
+
+	err := nm.addIPv6DefaultRoute()
+	if err == nil {
+		t.Fatal("Failed to test unhappy path with failing to add default route command")
+	}
+}