diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 66b25a33cd..7e6c57ed66 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -31,7 +31,7 @@ type MockIpamInvoker struct { delegatedVMNIC bool delegatedVMNICFail bool ipMap map[string]bool - customReturn map[string]network.InterfaceInfo + add func(opt IPAMAddConfig) (ipamAddResult IPAMAddResult, err error) } func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail bool) *MockIpamInvoker { @@ -47,8 +47,11 @@ func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail func NewCustomMockIpamInvoker(customReturn map[string]network.InterfaceInfo) *MockIpamInvoker { return &MockIpamInvoker{ - customReturn: customReturn, - + add: func(_ IPAMAddConfig) (ipamAddResult IPAMAddResult, err error) { + ipamAddResult = IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} + ipamAddResult.interfaceInfo = customReturn + return ipamAddResult, nil + }, ipMap: make(map[string]bool), } } @@ -112,9 +115,8 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes } } - if invoker.customReturn != nil { - ipamAddResult.interfaceInfo = invoker.customReturn - return ipamAddResult, nil + if invoker.add != nil { + return invoker.add(opt) } return ipamAddResult, nil diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 34ffc07042..2b090523c0 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -46,6 +46,7 @@ func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *net } } +// update epInfo data field, allow host to nc, allow nc to host, and network container id func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { logger.Info("Setting Endpoint Options") diff --git a/cni/network/network_linux_test.go b/cni/network/network_linux_test.go index b0fda74548..55bef48fa6 100644 --- a/cni/network/network_linux_test.go +++ b/cni/network/network_linux_test.go @@ -4,10 +4,18 @@ package network import ( + "fmt" + "net" + "regexp" "testing" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/platform" + "github.com/Azure/azure-container-networking/telemetry" + cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -160,3 +168,387 @@ func TestAddSnatForDns(t *testing.T) { }) } } + +// linux swiftv2 example +func GetTestCNSResponseSecondaryLinux(macAddress string) map[string]network.InterfaceInfo { + parsedMAC, _ := net.ParseMAC(macAddress) + return map[string]network.InterfaceInfo{ + string(cns.InfraNIC): { + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("20.241.0.35/16"), + Gateway: net.ParseIP("20.241.0.35"), // actual scenario doesn't have a gateway + }, + }, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("169.254.2.1/16"), + Gw: net.ParseIP("10.244.2.1"), + }, + { + Dst: *getCIDRNotationForAddress("0.0.0.0/32"), + Gw: net.ParseIP("169.254.2.1"), + }, + }, + NICType: cns.InfraNIC, + SkipDefaultRoutes: true, + HostSubnetPrefix: *getCIDRNotationForAddress("10.224.0.0/16"), + }, + macAddress: { + MacAddress: parsedMAC, + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.241.0.35/32"), + Gateway: net.ParseIP("10.241.0.35"), // actual scenario doesn't have a gateway + }, + }, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("169.254.2.1/32"), + Gw: net.ParseIP("10.244.2.1"), + }, + { + Dst: *getCIDRNotationForAddress("0.0.0.0/0"), + Gw: net.ParseIP("169.254.2.1"), + }, + }, + NICType: cns.NodeNetworkInterfaceFrontendNIC, + SkipDefaultRoutes: false, + }, + } +} + +// Happy path scenario for add and delete +func TestPluginLinuxAdd(t *testing.T) { + resources := GetTestResources() + mulNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "mulnet", + MultiTenancy: true, + EnableExactMatchForPodName: true, + Master: "eth0", + } + nwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "net", + MultiTenancy: false, + EnableExactMatchForPodName: true, + // test auto finding master interface + DNS: types.DNS{ + Nameservers: []string{ + "ns1", "ns2", + }, + Domain: "myDomain", + }, + } + macAddress := "60:45:bd76:f6:44" + parsedMACAddress, _ := net.ParseMAC(macAddress) + type endpointEntry struct { + epInfo *network.EndpointInfo + epIDRegex string + } + + tests := []struct { + name string + plugin *NetPlugin + args *cniSkel.CmdArgs + want []endpointEntry + match func(*network.EndpointInfo, *network.EndpointInfo) bool + }{ + { + // in swiftv1 linux multitenancy, we only get 1 response from cns at a time + name: "Add Happy Path Swiftv1 Multitenancy", + plugin: &NetPlugin{ + Plugin: resources.Plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + tb: &telemetry.TelemetryBuffer{}, + report: &telemetry.CNIReport{}, + multitenancyClient: NewMockMultitenancy(false, []*cns.GetNetworkContainerResponse{GetTestCNSResponse3()}), + }, + args: &cniSkel.CmdArgs{ + StdinData: mulNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + match: func(ei1, ei2 *network.EndpointInfo) bool { + return ei1.NetworkContainerID == ei2.NetworkContainerID + }, + want: []endpointEntry{ + // should match with GetTestCNSResponse3 + { + epInfo: &network.EndpointInfo{ + ContainerID: "test-container", + Data: map[string]interface{}{ + "VlanID": 1, // Vlan ID used here + "localIP": "168.254.0.4/17", + "snatBridgeIP": "168.254.0.1/17", + "vethname": "mulnettest-containereth0", + }, + Routes: []network.RouteInfo{ + { + Dst: *parseCIDR("192.168.0.4/24"), + Gw: net.ParseIP("192.168.0.1"), + // interface to use is NOT propagated to ep info + }, + }, + AllowInboundFromHostToNC: true, + EnableSnatOnHost: true, + EnableMultiTenancy: true, + EnableSnatForDns: true, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.InfraNIC, + MasterIfName: eth0IfName, + NetworkContainerID: "Swift_74b34111-6e92-49ee-a82a-8881c850ce0e", + NetworkID: "mulnet", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "20.240.0.0/24", + Options: map[string]interface{}{ + dockerNetworkOption: map[string]interface{}{ + "VlanID": "1", // doesn't seem to be used in linux + "snatBridgeIP": "168.254.0.1/17", + }, + }, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("20.0.0.10"), + Mask: getIPNetWithString("20.0.0.10/24").Mask, + }, + }, + NATInfo: nil, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + // matches cns ip configuration (20.0.0.1/24 == 20.0.0.0/24) + Prefix: *getIPNetWithString("20.0.0.0/24"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("20.0.0.1"), + }, + }, + }, + epIDRegex: `test-con-eth0`, + }, + }, + }, + { + // Based on a live swiftv2 linux cluster's cns invoker response + name: "Add Happy Path Swiftv2", + plugin: &NetPlugin{ + Plugin: resources.Plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + tb: &telemetry.TelemetryBuffer{}, + report: &telemetry.CNIReport{}, + ipamInvoker: &MockIpamInvoker{ + add: func(opt IPAMAddConfig) (ipamAddResult IPAMAddResult, err error) { + ipamAddResult = IPAMAddResult{interfaceInfo: make(map[string]network.InterfaceInfo)} + ipamAddResult.interfaceInfo = GetTestCNSResponseSecondaryLinux(macAddress) + opt.options["testflag"] = "copy" + return ipamAddResult, nil + }, + ipMap: make(map[string]bool), + }, + netClient: &InterfaceGetterMock{ + // used in secondary find master interface + interfaces: []net.Interface{ + { + Name: "secondary", + HardwareAddr: parsedMACAddress, + }, + { + Name: "primary", + HardwareAddr: net.HardwareAddr{}, + }, + }, + // used in primary find master interface + interfaceAddrs: map[string][]net.Addr{ + "primary": { + // match with the host subnet prefix to know that this ip belongs to the host + getCIDRNotationForAddress("10.224.0.0/16"), + }, + }, + }, + }, + args: &cniSkel.CmdArgs{ + StdinData: nwCfg.Serialize(), + ContainerID: "test-container", + Netns: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + match: func(ei1, ei2 *network.EndpointInfo) bool { + return ei1.NICType == ei2.NICType + }, + want: []endpointEntry{ + // should match infra + { + epInfo: &network.EndpointInfo{ + ContainerID: "test-container", + Data: map[string]interface{}{ + "vethname": "nettest-containereth0", + }, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("169.254.2.1/16"), + Gw: net.ParseIP("10.244.2.1"), + }, + { + Dst: *getCIDRNotationForAddress("0.0.0.0/32"), + Gw: net.ParseIP("169.254.2.1"), + }, + }, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.InfraNIC, + SkipDefaultRoutes: true, + MasterIfName: "primary", + NetworkID: "net", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "10.224.0.0/16", + EndpointDNS: network.DNSInfo{ + Servers: []string{ + "ns1", "ns2", + }, + Suffix: "myDomain", + }, + Options: map[string]interface{}{ + "testflag": "copy", + }, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("20.241.0.35"), + Mask: getIPNetWithString("20.241.0.35/16").Mask, + }, + }, + NATInfo: nil, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + // matches cns ip configuration (20.241.0.0/16 == 20.241.0.35/16) + Prefix: *getIPNetWithString("20.241.0.0/16"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("20.241.0.35"), + }, + }, + }, + epIDRegex: `.*`, + }, + // should match secondary + { + epInfo: &network.EndpointInfo{ + MacAddress: parsedMACAddress, + ContainerID: "test-container", + Data: map[string]interface{}{ + "vethname": "nettest-containereth0", + }, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("169.254.2.1/32"), + Gw: net.ParseIP("10.244.2.1"), + }, + { + Dst: *getCIDRNotationForAddress("0.0.0.0/0"), + Gw: net.ParseIP("169.254.2.1"), + }, + }, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.NodeNetworkInterfaceFrontendNIC, + SkipDefaultRoutes: false, + MasterIfName: "secondary", + NetworkID: "net", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "", + EndpointDNS: network.DNSInfo{ + Servers: []string{ + "ns1", "ns2", + }, + Suffix: "myDomain", + }, + Options: map[string]interface{}{ + "testflag": "copy", + }, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("10.241.0.35"), + Mask: getIPNetWithString("10.241.0.35/32").Mask, + }, + }, + NATInfo: nil, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + Prefix: *getIPNetWithString("10.241.0.35/32"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("10.241.0.35"), + }, + }, + }, + epIDRegex: `.*`, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Add(tt.args) + require.NoError(t, err) + allEndpoints, _ := tt.plugin.nm.GetAllEndpoints("") + require.Len(t, allEndpoints, len(tt.want)) + + // compare contents + for _, wantedEndpointEntry := range tt.want { + epID := "none" + for _, endpointInfo := range allEndpoints { + if !tt.match(wantedEndpointEntry.epInfo, endpointInfo) { + continue + } + // save the endpoint id before removing it + epID = endpointInfo.EndpointID + require.Regexp(t, regexp.MustCompile(wantedEndpointEntry.epIDRegex), epID) + + // omit endpoint id and ifname fields as they are nondeterministic + endpointInfo.EndpointID = "" + endpointInfo.IfName = "" + + require.Equal(t, wantedEndpointEntry.epInfo, endpointInfo) + } + if epID == "none" { + t.Fail() + } + err = tt.plugin.nm.DeleteEndpoint("", epID, nil) + require.NoError(t, err) + } + + // confirm separate entities + // that is, if one is modified, the other should not be modified + epInfos := []*network.EndpointInfo{} + for _, val := range allEndpoints { + epInfos = append(epInfos, val) + } + if len(epInfos) > 1 { + epInfo1 := epInfos[0] + epInfo2 := epInfos[1] + epInfo1.Data["dummy"] = "dummy value" + epInfo1.Options["dummy"] = "another dummy value" + require.NotEqual(t, epInfo1.Data, epInfo2.Data) + require.NotEqual(t, epInfo1.Options, epInfo2.Options) + } + + // ensure deleted + require.Empty(t, allEndpoints) + }) + } +} diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 060a1faaeb..f9bfa12084 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -688,6 +688,44 @@ func GetTestCNSResponse2() *cns.GetNetworkContainerResponse { } } +// For use with GetAllNetworkContainers in linux multitenancy +func GetTestCNSResponse3() *cns.GetNetworkContainerResponse { + return &cns.GetNetworkContainerResponse{ + NetworkContainerID: "Swift_74b34111-6e92-49ee-a82a-8881c850ce0e", + IPConfiguration: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "20.0.0.10", + PrefixLength: ipPrefixLen, + }, + DNSServers: []string{ + "168.63.129.16", + }, + GatewayIPAddress: "20.0.0.1", + }, + Routes: []cns.Route{ + // dummy route + { + IPAddress: "192.168.0.4/24", + GatewayIPAddress: "192.168.0.1", + }, + }, + MultiTenancyInfo: cns.MultiTenancyInfo{ + EncapType: cns.Vlan, + ID: multiTenancyVlan1, + }, + PrimaryInterfaceIdentifier: "20.240.0.4/24", + LocalIPConfiguration: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "168.254.0.4", + PrefixLength: localIPPrefixLen, + }, + GatewayIPAddress: "168.254.0.1", + }, + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + } +} + // Test Multitenancy Add func TestPluginMultitenancyAdd(t *testing.T) { plugin, _ := cni.NewPlugin("test", "0.3.0") diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index dbd2585aed..d912366a49 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "net" + "regexp" "testing" "github.com/Azure/azure-container-networking/cni" @@ -14,6 +15,7 @@ import ( "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/hnswrapper" "github.com/Azure/azure-container-networking/network/policy" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/telemetry" hnsv2 "github.com/Microsoft/hcsshim/hcn" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -855,3 +857,362 @@ func TestPluginMultitenancyWindowsDelete(t *testing.T) { }) } } + +// windows swiftv2 example +func GetTestCNSResponseSecondaryWindows(macAddress string) map[string]network.InterfaceInfo { + parsedMAC, _ := net.ParseMAC(macAddress) + return map[string]network.InterfaceInfo{ + string(cns.InfraNIC): { + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.244.2.107/16"), + Gateway: net.ParseIP("10.244.2.1"), + }, + }, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("1.1.1.1/24"), + Gw: net.ParseIP("10.244.2.1"), + }, + }, + SkipDefaultRoutes: true, + NICType: cns.InfraNIC, + HostSubnetPrefix: *getCIDRNotationForAddress("20.224.0.0/16"), + }, + macAddress: { + MacAddress: parsedMAC, + IPConfigs: []*network.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.241.0.21/16"), + Gateway: net.ParseIP("10.241.0.1"), + }, + }, + Routes: []network.RouteInfo{ + { + // just to ensure we don't overwrite if we had more routes + Dst: *getCIDRNotationForAddress("2.2.2.2/24"), + Gw: net.ParseIP("99.244.2.1"), + }, + }, + NICType: cns.NodeNetworkInterfaceFrontendNIC, + }, + } +} + +// Happy path scenario for add and delete +func TestPluginWindowsAdd(t *testing.T) { + resources := GetTestResources() + localNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "mulnet", + MultiTenancy: true, + EnableExactMatchForPodName: true, + Master: "eth0", + } + nwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "net", + MultiTenancy: false, + EnableExactMatchForPodName: true, + } + macAddress := "60:45:bd:76:f6:44" + parsedMACAddress, _ := net.ParseMAC(macAddress) + + type endpointEntry struct { + epInfo *network.EndpointInfo + epIDRegex string + } + + tests := []struct { + name string + plugin *NetPlugin + args *cniSkel.CmdArgs + want []endpointEntry + match func(*network.EndpointInfo, *network.EndpointInfo) bool + }{ + { + name: "Add Happy Path Dual NIC", + plugin: &NetPlugin{ + Plugin: resources.Plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + tb: &telemetry.TelemetryBuffer{}, + report: &telemetry.CNIReport{}, + multitenancyClient: NewMockMultitenancy(false, []*cns.GetNetworkContainerResponse{GetTestCNSResponse1(), GetTestCNSResponse2()}), + }, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + match: func(ei1, ei2 *network.EndpointInfo) bool { + return ei1.NetworkID == ei2.NetworkID + }, + want: []endpointEntry{ + // should match with GetTestCNSResponse1 + { + epInfo: &network.EndpointInfo{ + ContainerID: "test-container", + Data: map[string]interface{}{ + "cnetAddressSpace": []string(nil), + }, + Routes: []network.RouteInfo{}, + EnableSnatOnHost: true, + EnableMultiTenancy: true, + EnableSnatForDns: true, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.InfraNIC, + MasterIfName: eth0IfName, + NetworkID: "mulnet-vlan1-20-0-0-0_24", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "20.240.0.0/24", + Options: map[string]interface{}{ + dockerNetworkOption: map[string]interface{}{ + "VlanID": "1", + }, + }, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("20.0.0.10"), + Mask: getIPNetWithString("20.0.0.10/24").Mask, + }, + }, + // LocalIPConfiguration doesn't seem used in windows + // Constant, in windows, NAT Info comes from + // options > ipamAddConfig > + // cns invoker may populate network.SNATIPKey with the default response received > + // getNATInfo (with nwCfg) > adds nat info based on condition + // typically adds azure dns (168.63.129.16) + NATInfo: []policy.NATInfo{ + { + Destinations: []string{"168.63.129.16"}, + }, + }, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + // matches cns ip configuration (20.0.0.1/24 == 20.0.0.0/24) + Prefix: *getIPNetWithString("20.0.0.0/24"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("20.0.0.1"), + }, + }, + }, + epIDRegex: `.*`, + }, + // should match with GetTestCNSResponse2 + { + epInfo: &network.EndpointInfo{ + ContainerID: "test-container", + Data: map[string]interface{}{ + "cnetAddressSpace": []string(nil), + }, + Routes: []network.RouteInfo{}, + EnableSnatOnHost: true, + EnableMultiTenancy: true, + EnableSnatForDns: true, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.InfraNIC, + MasterIfName: eth0IfName, + NetworkID: "mulnet-vlan2-10-0-0-0_24", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "10.240.0.0/24", + Options: map[string]interface{}{ + dockerNetworkOption: map[string]interface{}{ + "VlanID": "2", + }, + }, + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("10.0.0.10"), + Mask: getIPNetWithString("10.0.0.10/24").Mask, + }, + }, + NATInfo: []policy.NATInfo{ + { + Destinations: []string{"168.63.129.16"}, + }, + }, + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + Prefix: *getIPNetWithString("10.0.0.0/24"), + Gateway: net.ParseIP("10.0.0.1"), + }, + }, + }, + epIDRegex: `.*`, + }, + }, + }, + { + // Based on a live swiftv2 windows cluster's (infra + delegated) cns invoker response + name: "Add Happy Path Swiftv2", + plugin: &NetPlugin{ + Plugin: resources.Plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + tb: &telemetry.TelemetryBuffer{}, + report: &telemetry.CNIReport{}, + ipamInvoker: NewCustomMockIpamInvoker(GetTestCNSResponseSecondaryWindows(macAddress)), + netClient: &InterfaceGetterMock{ + // used in secondary find master interface + interfaces: []net.Interface{ + { + Name: "secondary", + HardwareAddr: parsedMACAddress, + }, + { + Name: "primary", + HardwareAddr: net.HardwareAddr{}, + }, + }, + // used in primary find master interface + interfaceAddrs: map[string][]net.Addr{ + "primary": { + // match with the host subnet prefix to know that this ip belongs to the host + getCIDRNotationForAddress("20.224.0.0/16"), + }, + }, + }, + }, + args: &cniSkel.CmdArgs{ + StdinData: nwCfg.Serialize(), + ContainerID: "test-container", + Netns: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + match: func(ei1, ei2 *network.EndpointInfo) bool { + return ei1.NICType == ei2.NICType + }, + want: []endpointEntry{ + // should match infra + { + epInfo: &network.EndpointInfo{ + ContainerID: "test-container", + Data: map[string]interface{}{}, + Routes: []network.RouteInfo{ + { + Dst: *getCIDRNotationForAddress("1.1.1.1/24"), + Gw: net.ParseIP("10.244.2.1"), + }, + }, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.InfraNIC, + SkipDefaultRoutes: true, + MasterIfName: "primary", + NetworkID: "net", + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "20.224.0.0/16", + Options: map[string]interface{}{}, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("10.244.2.107"), + Mask: getIPNetWithString("10.244.2.107/16").Mask, + }, + }, + NATInfo: nil, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + Prefix: *getIPNetWithString("10.244.0.0/16"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("10.244.2.1"), + }, + }, + }, + epIDRegex: `.*`, + }, + // should match secondary + { + epInfo: &network.EndpointInfo{ + MacAddress: parsedMACAddress, + ContainerID: "test-container", + Data: map[string]interface{}{}, + Routes: []network.RouteInfo{ + { + // just to ensure we don't overwrite if we had more routes + Dst: *getCIDRNotationForAddress("2.2.2.2/24"), + Gw: net.ParseIP("99.244.2.1"), + }, + }, + PODName: "test-pod", + PODNameSpace: "test-pod-ns", + NICType: cns.NodeNetworkInterfaceFrontendNIC, + SkipDefaultRoutes: false, + MasterIfName: "secondary", + NetworkID: "azure-" + macAddress, + NetNsPath: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + NetNs: "bc526fae-4ba0-4e80-bc90-ad721e5850bf", + HostSubnetPrefix: "", + Options: map[string]interface{}{}, + // matches with cns ip configuration + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("10.241.0.21"), + Mask: getIPNetWithString("10.241.0.21/16").Mask, + }, + }, + NATInfo: nil, + // ip config pod ip + mask(s) from cns > interface info > subnet info + Subnets: []network.SubnetInfo{ + { + Family: platform.AfINET, + Prefix: *getIPNetWithString("10.241.0.21/16"), + // matches cns ip configuration gateway ip address + Gateway: net.ParseIP("10.241.0.1"), + }, + }, + }, + epIDRegex: `.*`, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Add(tt.args) + require.NoError(t, err) + allEndpoints, _ := tt.plugin.nm.GetAllEndpoints("") + require.Len(t, allEndpoints, len(tt.want)) + for _, wantedEndpointEntry := range tt.want { + epID := "none" + for _, endpointInfo := range allEndpoints { + if !tt.match(wantedEndpointEntry.epInfo, endpointInfo) { + continue + } + // save the endpoint id before removing it + epID = endpointInfo.EndpointID + require.Regexp(t, regexp.MustCompile(wantedEndpointEntry.epIDRegex), epID) + + // omit endpoint id and ifname fields as they are nondeterministic + endpointInfo.EndpointID = "" + endpointInfo.IfName = "" + + require.Equal(t, wantedEndpointEntry.epInfo, endpointInfo) + } + if epID == "none" { + t.Fail() + } + err = tt.plugin.nm.DeleteEndpoint("", epID, nil) + require.NoError(t, err) + } + + // ensure deleted + require.Empty(t, allEndpoints) + }) + } +} diff --git a/network/endpoint.go b/network/endpoint.go index e5b55a0b36..0dc122ce9c 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -66,7 +66,7 @@ type EndpointInfo struct { EndpointID string ContainerID string NetNsPath string - IfName string // value differs during creation vs. deletion flow + IfName string // value differs during creation vs. deletion flow; used in statefile, not necessarily the nic name SandboxKey string IfIndex int MacAddress net.HardwareAddr @@ -93,7 +93,7 @@ type EndpointInfo struct { IPV6Mode string VnetCidrs string ServiceCidrs string - NATInfo []policy.NATInfo + NATInfo []policy.NATInfo // windows only NICType cns.NICType SkipDefaultRoutes bool HNSEndpointID string diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 150efd1dea..a0e4b6c708 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -79,6 +79,7 @@ func TestSecondaryAddEndpoints(t *testing.T) { } else { require.NoError(t, err) require.Equal(t, tt.client.ep.SecondaryInterfaces["eth1"].MacAddress, tt.epInfo.MacAddress) + require.Equal(t, "eth1", tt.epInfo.IfName, "interface name should update based on mac address here before being referenced later") } }) }