Merge branch 'sanprabhu/cilium-node-subnet-nc' of github.com:Azure/azure-container-networking into sanprabhu/cilium-node-subnet-nc

Author: santhoshmprabhu

cns/NetworkContainerContract.go

@@ -55,6 +55,7 @@ const (
 	JobObject              = "JobObject"
 	COW                    = "COW" // Container on Windows
 	BackendNICNC           = "BackendNICNC"
+	NodeSubnet             = "NodeSubnet" // Artificial Container Type for NodeSubnet
 )
 
 // Orchestrator Types
@@ -66,6 +67,7 @@ const (
 	AzureFirstParty = "AzureFirstParty"
 	KubernetesCRD   = "KubernetesCRD"
 	// TODO: Add OrchastratorType as CRD: https://msazure.visualstudio.com/One/_workitems/edit/7711872
+	KubernetesNodeSubnet = "KubernetesNodeSubnet"
 )
 
 // Encap Types
@@ -99,10 +101,13 @@ const (
 	Managed        = "Managed"
 	CRD            = "CRD"
 	MultiTenantCRD = "MultiTenantCRD"
+	AzureHost      = "AzureHost"
 )
 
-var ErrInvalidNCID = errors.New("invalid NetworkContainerID")
-var ErrInvalidIP = errors.New("invalid IP")
+var (
+	ErrInvalidNCID = errors.New("invalid NetworkContainerID")
+	ErrInvalidIP   = errors.New("invalid IP")
+)
 
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.
 type CreateNetworkContainerRequest struct {

cns/ipam/interfaces.go

@@ -0,0 +1,11 @@
+package ipam
+
+import (
+	"github.com/Azure/azure-container-networking/cns"
+	cnstypes "github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+)
+
+type IpamStateReconciler interface {
+	ReconcileIPAMState(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo, nnc *v1alpha.NodeNetworkConfig) cnstypes.ResponseCode
+}

cns/nodesubnet/initialization.go

@@ -0,0 +1,33 @@
+package nodesubnet
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/ipam"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
+)
+
+func ReconcileInitialCNSState(ctx context.Context, ipamReconciler ipam.IpamStateReconciler, podInfoByIPProvider cns.PodInfoByIPProvider) (int, error) {
+	// Get previous PodInfo state from podInfoByIPProvider
+	podInfoByIP, err := podInfoByIPProvider.PodInfoByIP()
+	if err != nil {
+		return 0, errors.Wrap(err, "provider failed to provide PodInfoByIP")
+	}
+
+	logger.Printf("Reconciling initial CNS state with %d IPs", len(podInfoByIP))
+
+	// Create a network container request that holds all the IPs from PodInfoByIP
+	secondaryIPs := maps.Keys(podInfoByIP)
+	ncRequest := CreateNodeSubnetNCRequest(secondaryIPs)
+	responseCode := ipamReconciler.ReconcileIPAMState([]*cns.CreateNetworkContainerRequest{ncRequest}, podInfoByIP, nil)
+
+	if responseCode != types.Success {
+		return 0, errors.Errorf("failed to reconcile initial CNS state: %d", responseCode)
+	}
+
+	return len(secondaryIPs), nil
+}

cns/nodesubnet/initialization_test.go

@@ -0,0 +1,110 @@
+package nodesubnet_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/cnireconciler"
+	"github.com/Azure/azure-container-networking/cns/ipam"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	"github.com/Azure/azure-container-networking/store"
+)
+
+func getMockStore() store.KeyValueStore {
+	mockStore := store.NewMockStore("")
+	endpointState := map[string]*restserver.EndpointInfo{
+		"12e65d89e58cb23c784e97840cf76866bfc9902089bdc8e87e9f64032e312b0b": {
+			PodName:      "coredns-54b69f46b8-ldmwr",
+			PodNamespace: "kube-system",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 52),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+		"1fc5176913a3a1a7facfb823dde3b4ded404041134fef4f4a0c8bba140fc0413": {
+			PodName:      "load-test-7f7d49687d-wxc9p",
+			PodNamespace: "load-test",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 63),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := mockStore.Write(restserver.EndpointStoreKey, endpointState)
+	if err != nil {
+		return nil
+	}
+	return mockStore
+}
+
+type MockIpamStateReconciler struct{}
+
+func (m *MockIpamStateReconciler) ReconcileIPAMState(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo, nnc *v1alpha.NodeNetworkConfig) types.ResponseCode {
+	if len(ncRequests) == 1 && len(ncRequests[0].SecondaryIPConfigs) == len(podInfoByIP) {
+		return types.Success
+	}
+
+	return types.UnexpectedError
+}
+
+func TestNewCNSPodInfoProvider(t *testing.T) {
+	tests := []struct {
+		name       string
+		store      store.KeyValueStore
+		wantErr    bool
+		reconciler ipam.IpamStateReconciler
+		exp        int
+	}{
+		{
+			name:       "happy_path",
+			store:      getMockStore(),
+			wantErr:    false,
+			reconciler: &MockIpamStateReconciler{},
+			exp:        2,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		ctx, cancel := testContext(t)
+		defer cancel()
+
+		t.Run(tt.name, func(t *testing.T) {
+			podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(tt.store)
+			checkErr(t, err, false)
+
+			got, err := nodesubnet.ReconcileInitialCNSState(ctx, tt.reconciler, podInfoByIPProvider)
+			checkErr(t, err, tt.wantErr)
+			if got != tt.exp {
+				t.Errorf("got %d IPs reconciled, expected %d", got, tt.exp)
+			}
+		})
+	}
+}
+
+// testContext creates a context from the provided testing.T that will be
+// canceled if the test suite is terminated.
+func testContext(t *testing.T) (context.Context, context.CancelFunc) {
+	if deadline, ok := t.Deadline(); ok {
+		return context.WithDeadline(context.Background(), deadline)
+	}
+	return context.WithCancel(context.Background())
+}

cns/restserver/internalapi.go

@@ -291,6 +291,7 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 		}
 	}
 
+	logger.Printf("Saved NC")
 	// index all the secondary IP configs for all the nc reqs, for easier lookup later on.
 	allSecIPsIdx := make(map[string]*cns.CreateNetworkContainerRequest)
 	for i := range ncReqs {
@@ -299,6 +300,8 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 		}
 	}
 
+	logger.Printf("0")
+
 	// we now need to reconcile IP assignment.
 	// considering that a single pod may have multiple ips (such as in dual stack scenarios)
 	// and that IP assignment in CNS (as done by requestIPConfigsHelper) does not allow
@@ -327,7 +330,10 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 		return types.UnexpectedError
 	}
 
+	logger.Printf("1")
+
 	for podKey, podIPs := range podKeyToPodIPs {
+		logger.Printf("Pod is %s", podKey)
 		var (
 			desiredIPs []string
 			ncIDs      []string
@@ -378,9 +384,11 @@ func (service *HTTPRestService) ReconcileIPAMState(ncReqs []*cns.CreateNetworkCo
 		}
 	}
 
-	if err := service.MarkExistingIPsAsPendingRelease(nnc.Spec.IPsNotInUse); err != nil {
-		logger.Errorf("[Azure CNS] Error. Failed to mark IPs as pending %v", nnc.Spec.IPsNotInUse)
-		return types.UnexpectedError
+	if nnc != nil {
+		if err := service.MarkExistingIPsAsPendingRelease(nnc.Spec.IPsNotInUse); err != nil {
+			logger.Errorf("[Azure CNS] Error. Failed to mark IPs as pending %v", nnc.Spec.IPsNotInUse)
+			return types.UnexpectedError
+		}
 	}
 
 	return 0
@@ -521,16 +529,24 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns.
 
 	// For now only RequestController uses this API which will be initialized only for AKS scenario.
 	// Validate ContainerType is set as Docker
-	if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes {
+	if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes && service.state.OrchestratorType != cns.KubernetesNodeSubnet {
 		logger.Errorf("[Azure CNS] Error. Unsupported OrchestratorType: %s", service.state.OrchestratorType)
 		return types.UnsupportedOrchestratorType
 	}
 
-	// Validate PrimaryCA must never be empty
-	err := validateIPSubnet(req.IPConfiguration.IPSubnet)
-	if err != nil {
-		logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
-		return types.InvalidPrimaryIPConfig
+	if req.NetworkContainerType == cns.NodeSubnet {
+		// For NodeSubnet scenarios, Validate PrimaryCA must be empty
+		if req.IPConfiguration.IPSubnet.IPAddress != "" {
+			logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
+			return types.InvalidPrimaryIPConfig
+		}
+	} else {
+		// For Swift scenarios, Validate PrimaryCA must never be empty
+		err := validateIPSubnet(req.IPConfiguration.IPSubnet)
+		if err != nil {
+			logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req)
+			return types.InvalidPrimaryIPConfig
+		}
 	}
 
 	// Validate SecondaryIPConfig