Merge branch 'master' into isaiahraya/npm-cilium-migration-script

Author: rayaisaiah

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

@@ -113,38 +113,4 @@ steps:
       displayName: "Windows DualStack Overlay Datapath Tests"
       retryCountOnTaskFailure: 3
 
-    - task: AzureCLI@1
-      inputs:
-        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-        scriptLocation: "inlineScript"
-        scriptType: "bash"
-        addSpnToEnvironment: true
-        inlineScript: |
-          set -e
-          clusterName=${{ parameters.clusterName }}
-          echo "Restarting nodes"
-          for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do
-            make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val}
-          done
-      displayName: "Restart Nodes"
-
-    - task: AzureCLI@1
-      inputs:
-        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-        scriptLocation: "inlineScript"
-        scriptType: "bash"
-        addSpnToEnvironment: true
-        inlineScript: |
-          set -e
-          cd test/integration/load
-
-          # Scale Cluster Up/Down to confirm functioning CNS
-          ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
-          kubectl get pods -owide -A
-
-          cd ../../..
-          echo "Validating Node Restart"
-          make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
-          kubectl delete ns load-test
-      displayName: "Validate Node Restart"
-      retryCountOnTaskFailure: 3
+    # Windows node restart and validation tests removed due to flakiness

cns/client/client_test.go

@@ -152,7 +152,7 @@ func TestMain(m *testing.M) {
 	config := common.ServiceConfig{}
 
 	httpRestService, err := restserver.NewHTTPRestService(&config, &fakes.WireserverClientFake{},
-		&fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, nil, nil, nil,
+		&fakes.WireserverProxyFake{}, &restserver.IPtablesProvider{}, &fakes.NMAgentClientFake{}, nil, nil, nil,
 		fakes.NewMockIMDSClient())
 	svc = httpRestService
 	httpRestService.Name = "cns-test-server"

cns/fakes/iptablesfake.go

@@ -0,0 +1,103 @@
+package fakes
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/Azure/azure-container-networking/iptables"
+)
+
+var (
+	errChainExists   = errors.New("chain already exists")
+	errChainNotFound = errors.New("chain not found")
+	errRuleExists    = errors.New("rule already exists")
+)
+
+type IPTablesMock struct {
+	state map[string]map[string][]string
+}
+
+func NewIPTablesMock() *IPTablesMock {
+	return &IPTablesMock{
+		state: make(map[string]map[string][]string),
+	}
+}
+
+func (c *IPTablesMock) ensureTableExists(table string) {
+	_, exists := c.state[table]
+	if !exists {
+		c.state[table] = make(map[string][]string)
+	}
+}
+
+func (c *IPTablesMock) ChainExists(table, chain string) (bool, error) {
+	c.ensureTableExists(table)
+
+	builtins := []string{iptables.Input, iptables.Output, iptables.Prerouting, iptables.Postrouting, iptables.Forward}
+
+	_, exists := c.state[table][chain]
+
+	// these chains always exist
+	for _, val := range builtins {
+		if chain == val && !exists {
+			c.state[table][chain] = []string{}
+			return true, nil
+		}
+	}
+
+	return exists, nil
+}
+
+func (c *IPTablesMock) NewChain(table, chain string) error {
+	c.ensureTableExists(table)
+
+	exists, _ := c.ChainExists(table, chain)
+
+	if exists {
+		return errChainExists
+	}
+
+	c.state[table][chain] = []string{}
+	return nil
+}
+
+func (c *IPTablesMock) Exists(table, chain string, rulespec ...string) (bool, error) {
+	c.ensureTableExists(table)
+
+	chainExists, _ := c.ChainExists(table, chain)
+	if !chainExists {
+		return false, nil
+	}
+
+	targetRule := strings.Join(rulespec, " ")
+	chainRules := c.state[table][chain]
+
+	for _, chainRule := range chainRules {
+		if targetRule == chainRule {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (c *IPTablesMock) Append(table, chain string, rulespec ...string) error {
+	c.ensureTableExists(table)
+
+	chainExists, _ := c.ChainExists(table, chain)
+	if !chainExists {
+		return errChainNotFound
+	}
+
+	ruleExists, _ := c.Exists(table, chain, rulespec...)
+	if ruleExists {
+		return errRuleExists
+	}
+
+	targetRule := strings.Join(rulespec, " ")
+	c.state[table][chain] = append(c.state[table][chain], targetRule)
+	return nil
+}
+
+func (c *IPTablesMock) Insert(table, chain string, _ int, rulespec ...string) error {
+	return c.Append(table, chain, rulespec...)
+}

cns/restserver/api_test.go

@@ -1735,7 +1735,7 @@ func startService(serviceConfig common.ServiceConfig, _ configuration.CNSConfig)
 	config.Store = fileStore
 
 	nmagentClient := &fakes.NMAgentClientFake{}
-	service, err = NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{},
+	service, err = NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &IPtablesProvider{},
 		nmagentClient, nil, nil, nil, fakes.NewMockIMDSClient())
 	if err != nil {
 		return err

cns/restserver/internalapi_linux.go

@@ -11,18 +11,28 @@ import (
 	"github.com/Azure/azure-container-networking/iptables"
 	"github.com/Azure/azure-container-networking/network/networkutils"
 	goiptables "github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
 )
 
 const SWIFT = "SWIFT-POSTROUTING"
 
+type IPtablesProvider struct{}
+
+func (c *IPtablesProvider) GetIPTables() (iptablesClient, error) {
+	client, err := goiptables.New()
+	return client, errors.Wrap(err, "failed to get iptables client")
+}
+
 // nolint
 func (service *HTTPRestService) programSNATRules(req *cns.CreateNetworkContainerRequest) (types.ResponseCode, string) {
 	service.Lock()
 	defer service.Unlock()
 
 	// Parse primary ip and ipnet from nnc
-	ncPrimaryIP, ncIPNet, _ := net.ParseCIDR(req.IPConfiguration.IPSubnet.IPAddress + "/" + fmt.Sprintf("%d", req.IPConfiguration.IPSubnet.PrefixLength))
-	ipt, err := goiptables.New()
+	// in podsubnet case, ncPrimaryIP is the pod subnet's primary ip
+	// in vnet scale case, ncPrimaryIP is the node's ip
+	ncPrimaryIP, _, _ := net.ParseCIDR(req.IPConfiguration.IPSubnet.IPAddress + "/" + fmt.Sprintf("%d", req.IPConfiguration.IPSubnet.PrefixLength))
+	ipt, err := service.iptables.GetIPTables()
 	if err != nil {
 		return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to create iptables interface : %v", err)
 	}
@@ -56,41 +66,51 @@ func (service *HTTPRestService) programSNATRules(req *cns.CreateNetworkContainer
 		}
 	}
 
-	snatUDPRuleexist, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureDNS, "-p", iptables.UDP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
-	if err != nil {
-		return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of SNAT UDP rule : %v", err)
-	}
-	if !snatUDPRuleexist {
-		logger.Printf("[Azure CNS] Inserting SNAT UDP rule ...")
-		err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureDNS, "-p", iptables.UDP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
+	// use any secondary ip + the nnc prefix length to get an iptables rule to allow dns and imds traffic from the pods
+	for _, v := range req.SecondaryIPConfigs {
+		// put the ip address in standard cidr form (where we zero out the parts that are not relevant)
+		_, podSubnet, _ := net.ParseCIDR(v.IPAddress + "/" + fmt.Sprintf("%d", req.IPConfiguration.IPSubnet.PrefixLength))
+
+		snatUDPRuleExists, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureDNS, "-p", iptables.UDP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
 		if err != nil {
-			return types.FailedToRunIPTableCmd, "[Azure CNS] failed to inset SNAT UDP rule : " + err.Error()
+			return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of pod SNAT UDP rule : %v", err)
+		}
+		if !snatUDPRuleExists {
+			logger.Printf("[Azure CNS] Inserting pod SNAT UDP rule ...")
+			err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureDNS, "-p", iptables.UDP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
+			if err != nil {
+				return types.FailedToRunIPTableCmd, "[Azure CNS] failed to insert pod SNAT UDP rule : " + err.Error()
+			}
 		}
-	}
 
-	snatTCPRuleexist, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureDNS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
-	if err != nil {
-		return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of SNAT TCP rule : %v", err)
-	}
-	if !snatTCPRuleexist {
-		logger.Printf("[Azure CNS] Inserting SNAT TCP rule ...")
-		err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureDNS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
+		snatPodTCPRuleExists, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureDNS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
 		if err != nil {
-			return types.FailedToRunIPTableCmd, "[Azure CNS] failed to insert SNAT TCP rule : " + err.Error()
+			return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of pod SNAT TCP rule : %v", err)
+		}
+		if !snatPodTCPRuleExists {
+			logger.Printf("[Azure CNS] Inserting pod SNAT TCP rule ...")
+			err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureDNS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.DNSPort), "-j", iptables.Snat, "--to", ncPrimaryIP.String())
+			if err != nil {
+				return types.FailedToRunIPTableCmd, "[Azure CNS] failed to insert pod SNAT TCP rule : " + err.Error()
+			}
 		}
-	}
 
-	snatIMDSRuleexist, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureIMDS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.HTTPPort), "-j", iptables.Snat, "--to", req.HostPrimaryIP)
-	if err != nil {
-		return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of SNAT IMDS rule : %v", err)
-	}
-	if !snatIMDSRuleexist {
-		logger.Printf("[Azure CNS] Inserting SNAT IMDS rule ...")
-		err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", ncIPNet.String(), "-d", networkutils.AzureIMDS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.HTTPPort), "-j", iptables.Snat, "--to", req.HostPrimaryIP)
+		snatIMDSRuleexist, err := ipt.Exists(iptables.Nat, SWIFT, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureIMDS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.HTTPPort), "-j", iptables.Snat, "--to", req.HostPrimaryIP)
 		if err != nil {
-			return types.FailedToRunIPTableCmd, "[Azure CNS] failed to insert SNAT IMDS rule : " + err.Error()
+			return types.UnexpectedError, fmt.Sprintf("[Azure CNS] Error. Failed to check for existence of pod SNAT IMDS rule : %v", err)
+		}
+		if !snatIMDSRuleexist {
+			logger.Printf("[Azure CNS] Inserting pod SNAT IMDS rule ...")
+			err = ipt.Insert(iptables.Nat, SWIFT, 1, "-m", "addrtype", "!", "--dst-type", "local", "-s", podSubnet.String(), "-d", networkutils.AzureIMDS, "-p", iptables.TCP, "--dport", strconv.Itoa(iptables.HTTPPort), "-j", iptables.Snat, "--to", req.HostPrimaryIP)
+			if err != nil {
+				return types.FailedToRunIPTableCmd, "[Azure CNS] failed to insert pod SNAT IMDS rule : " + err.Error()
+			}
 		}
+
+		// we only need to run this code once as the iptable rule applies to all secondary ip configs in the same subnet
+		break
 	}
+
 	return types.Success, ""
 }