feat: support for cilium + nodesubnet

Author: santhoshmprabhu

.pipelines/pipeline.yaml

@@ -261,6 +261,17 @@ stages:
         k8sVersion: ""
         dependsOn: "containerize"
 
+    # Cilium Nodesubnet E2E tests
+    - template: singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml
+      parameters:
+        name: "cilium_nodesubnet_e2e"
+        displayName: Cilium NodeSubnet
+        clusterType: nodesubnet-byocni-nokubeproxy-up
+        clusterName: "cilndsubnete2e"
+        vmSize: Standard_B2s
+        k8sVersion: ""
+        dependsOn: "containerize"
+
     # Cilium Overlay E2E tests
     - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
       parameters:
@@ -405,6 +416,7 @@ stages:
         - azure_overlay_stateless_e2e
         - aks_swift_e2e
         - cilium_e2e
+        - cilium_nodesubnet_e2e
         - cilium_overlay_e2e
         - cilium_h_overlay_e2e
         - aks_ubuntu_22_linux_e2e
@@ -425,6 +437,10 @@ stages:
               cilium_e2e:
                 name: cilium_e2e
                 clusterName: "ciliume2e"
+                region: $(REGION_AKS_CLUSTER_TEST)                
+              cilium_nodesubnet_e2e:
+                name: cilium_nodesubnet_e2e
+                clusterName: "cilndsubnete2e"
                 region: $(REGION_AKS_CLUSTER_TEST)
               cilium_overlay_e2e:
                 name: cilium_overlay_e2e

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml

@@ -71,9 +71,9 @@ stages:
           os: ${{ parameters.os }}
           datapath: true
           dns: true
+          cni: cilium
           portforward: true
           service: true
-          hostport: true
           dependsOn: ${{ parameters.name }}
 
       - job: failedE2ELogs

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml

@@ -57,6 +57,7 @@ steps:
         pwd
         kubectl cluster-info
         kubectl get po -owide -A
+        CILIUM_VERSION_TAG=${CILIUM_NODE_SUBNET_VERSION}
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         export DIR=${CILIUM_VERSION_TAG%.*}
         echo "installing files from ${DIR}"

cns/fakes/nmagentclientfake.go

@@ -14,9 +14,10 @@ import (
 
 // NMAgentClientFake can be used to query to VM Host info.
 type NMAgentClientFake struct {
-	SupportedAPIsF    func(context.Context) ([]string, error)
-	GetNCVersionListF func(context.Context) (nmagent.NCVersionList, error)
-	GetHomeAzF        func(context.Context) (nmagent.AzResponse, error)
+	SupportedAPIsF      func(context.Context) ([]string, error)
+	GetNCVersionListF   func(context.Context) (nmagent.NCVersionList, error)
+	GetHomeAzF          func(context.Context) (nmagent.AzResponse, error)
+	GetInterfaceIPInfoF func(ctx context.Context) (nmagent.Interfaces, error)
 }
 
 func (n *NMAgentClientFake) SupportedAPIs(ctx context.Context) ([]string, error) {
@@ -30,3 +31,7 @@ func (n *NMAgentClientFake) GetNCVersionList(ctx context.Context) (nmagent.NCVer
 func (n *NMAgentClientFake) GetHomeAz(ctx context.Context) (nmagent.AzResponse, error) {
 	return n.GetHomeAzF(ctx)
 }
+
+func (n *NMAgentClientFake) GetInterfaceIPInfo(ctx context.Context) (nmagent.Interfaces, error) {
+	return n.GetInterfaceIPInfoF(ctx)
+}

cns/restserver/nodesubnet.go

@@ -0,0 +1,56 @@
+package restserver
+
+import (
+	"context"
+	"net/netip"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	nodesubnet "github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/cns/types"
+	errors "github.com/pkg/errors"
+)
+
+var _ nodesubnet.IPConsumer = &HTTPRestService{}
+
+// Implement the UpdateIPsForNodeSubnet method for HTTPRestService
+func (service *HTTPRestService) UpdateIPsForNodeSubnet(secondaryIPs []netip.Addr) error {
+	secondaryIPStrs := make([]string, len(secondaryIPs))
+	for i, ip := range secondaryIPs {
+		secondaryIPStrs[i] = ip.String()
+	}
+
+	networkContainerRequest := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPStrs)
+
+	code, msg := service.saveNetworkContainerGoalState(*networkContainerRequest)
+	if code != types.Success {
+		logger.Debugf("Error in processing IP change")
+		return errors.Errorf("failed to save fetched ips. code: %d, message %s", code, msg)
+	} else {
+		logger.Debugf("IP change processed successfully")
+	}
+
+	// saved NC successfully, generate conflist to indicate CNS is ready
+	go service.MustGenerateCNIConflistOnce()
+	return nil
+}
+
+func (service *HTTPRestService) InitializeNodeSubnet(ctx context.Context, podInfoByIPProvider cns.PodInfoByIPProvider) error {
+	// Set orchestrator type
+	orchestrator := cns.SetOrchestratorTypeRequest{
+		OrchestratorType: cns.KubernetesCRD,
+	}
+	service.SetNodeOrchestrator(&orchestrator)
+	service.nodesubnetIPFetcher = nodesubnet.NewIPFetcher(service.nma, service, 0, 0, logger.Log)
+	if podInfoByIPProvider == nil {
+		logger.Printf("PodInfoByIPProvider is nil, this usually means no saved endpoint state. Skipping reconciliation")
+	} else if _, err := nodesubnet.ReconcileInitialCNSState(ctx, service, podInfoByIPProvider); err != nil {
+		return errors.Wrap(err, "reconcile initial CNS state")
+	}
+
+	return nil
+}
+
+func (service *HTTPRestService) StartNodeSubnet(ctx context.Context) {
+	service.nodesubnetIPFetcher.Start(ctx)
+}

cns/restserver/restserver.go

@@ -13,11 +13,13 @@ import (
 	"github.com/Azure/azure-container-networking/cns/dockerclient"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/networkcontainers"
+	nodesubnet "github.com/Azure/azure-container-networking/cns/nodesubnet"
 	"github.com/Azure/azure-container-networking/cns/routes"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/cns/types/bounded"
 	"github.com/Azure/azure-container-networking/cns/wireserver"
 	acn "github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/nmagent"
 	nma "github.com/Azure/azure-container-networking/nmagent"
 	"github.com/Azure/azure-container-networking/store"
 	"github.com/pkg/errors"
@@ -40,6 +42,7 @@ type nmagentClient interface {
 	SupportedAPIs(context.Context) ([]string, error)
 	GetNCVersionList(context.Context) (nma.NCVersionList, error)
 	GetHomeAz(context.Context) (nma.AzResponse, error)
+	GetInterfaceIPInfo(ctx context.Context) (nmagent.Interfaces, error)
 }
 
 type wireserverProxy interface {
@@ -76,6 +79,7 @@ type HTTPRestService struct {
 	IPConfigsHandlerMiddleware cns.IPConfigsHandlerMiddleware
 	PnpIDByMacAddress          map[string]string
 	imdsClient                 imdsClient
+	nodesubnetIPFetcher        *nodesubnet.IPFetcher
 }
 
 type CNIConflistGenerator interface {

cns/service/main.go

@@ -661,6 +661,8 @@ func main() {
 	}
 	if isManaged, ok := acn.GetArg(acn.OptManaged).(bool); ok && isManaged {
 		config.ChannelMode = cns.Managed
+	} else if cnsconfig.ChannelMode == cns.AzureHost {
+		config.ChannelMode = cns.AzureHost
 	}
 
 	homeAzMonitor := restserver.NewHomeAzMonitor(nmaClient, time.Duration(cnsconfig.AZRSettings.PopulateHomeAzCacheRetryIntervalSecs)*time.Second)
@@ -736,7 +738,6 @@ func main() {
 	}
 
 	imdsClient := imds.NewClient()
-
 	httpRemoteRestService, err := restserver.NewHTTPRestService(&config, wsclient, &wsProxy, nmaClient,
 		endpointStateStore, conflistGenerator, homeAzMonitor, imdsClient)
 	if err != nil {
@@ -871,6 +872,26 @@ func main() {
 		}
 	}
 
+	if config.ChannelMode == cns.AzureHost {
+		if !cnsconfig.ManageEndpointState {
+			logger.Errorf("[Azure CNS] ManageEndpointState must be set to true for AzureHost mode")
+			return
+		} else {
+			// If cns manageendpointstate is true, then cns maintains its own state and reconciles from it.
+			// in this case, cns maintains state with containerid as key and so in-memory cache can lookup
+			// and update based on container id.
+			cns.GlobalPodInfoScheme = cns.InfraIDPodInfoScheme
+		}
+
+		podInfoByIPProvider, err := getPodInfoByIPProvider(rootCtx, cnsconfig, httpRemoteRestService, nil, "")
+		if err != nil {
+			logger.Errorf("[Azure CNS] Failed to get PodInfoByIPProvider: %v", err)
+			return
+		}
+
+		httpRemoteRestService.InitializeNodeSubnet(rootCtx, podInfoByIPProvider)
+	}
+
 	// Initialize multi-tenant controller if the CNS is running in MultiTenantCRD mode.
 	// It must be started before we start HTTPRemoteRestService.
 	if config.ChannelMode == cns.MultiTenantCRD {
@@ -909,6 +930,7 @@ func main() {
 	}
 
 	// if user provides cns url by -c option, then only start HTTP remote server using this url
+
 	logger.Printf("[Azure CNS] Start HTTP Remote server")
 	if httpRemoteRestService != nil {
 		if cnsconfig.EnablePprof {
@@ -1019,6 +1041,10 @@ func main() {
 		}(privateEndpoint, infravnet, nodeID)
 	}
 
+	if config.ChannelMode == cns.AzureHost {
+		httpRemoteRestService.StartNodeSubnet(rootCtx)
+	}
+
 	// mark the service as "ready"
 	close(readyCh)
 	// block until process exiting
@@ -1249,40 +1275,11 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		}
 	}
 
-	var podInfoByIPProvider cns.PodInfoByIPProvider
-	switch {
-	case cnsconfig.ManageEndpointState:
-		logger.Printf("Initializing from self managed endpoint store")
-		podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server
-		if err != nil {
-			if errors.Is(err, store.ErrKeyNotFound) {
-				logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state")
-			} else {
-				return errors.Wrap(err, "failed to create CNS PodInfoProvider")
-			}
-		}
-	case cnsconfig.InitializeFromCNI:
-		logger.Printf("Initializing from CNI")
-		podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider()
-		if err != nil {
-			return errors.Wrap(err, "failed to create CNI PodInfoProvider")
-		}
-	default:
-		logger.Printf("Initializing from Kubernetes")
-		podInfoByIPProvider = cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) {
-			pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{ //nolint:govet // ignore err shadow
-				FieldSelector: "spec.nodeName=" + nodeName,
-			})
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to list Pods for PodInfoProvider")
-			}
-			podInfo, err := cns.KubePodsToPodInfoByIP(pods.Items)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to convert Pods to PodInfoByIP")
-			}
-			return podInfo, nil
-		})
+	podInfoByIPProvider, err := getPodInfoByIPProvider(ctx, cnsconfig, httpRestServiceImplementation, clientset, nodeName)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize ip state")
 	}
+
 	// create scoped kube clients.
 	directcli, err := client.New(kubeConfig, client.Options{Scheme: nodenetworkconfig.Scheme})
 	if err != nil {
@@ -1505,6 +1502,44 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	return nil
 }
 
+func getPodInfoByIPProvider(ctx context.Context, cnsconfig *configuration.CNSConfig, httpRestServiceImplementation *restserver.HTTPRestService, clientset *kubernetes.Clientset, nodeName string) (podInfoByIPProvider cns.PodInfoByIPProvider, err error) {
+	switch {
+	case cnsconfig.ManageEndpointState:
+		logger.Printf("Initializing from self managed endpoint store")
+		podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server
+		if err != nil {
+			if errors.Is(err, store.ErrKeyNotFound) {
+				logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state")
+			} else {
+				return podInfoByIPProvider, errors.Wrap(err, "failed to create CNS PodInfoProvider")
+			}
+		}
+	case cnsconfig.InitializeFromCNI:
+		logger.Printf("Initializing from CNI")
+		podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider()
+		if err != nil {
+			return podInfoByIPProvider, errors.Wrap(err, "failed to create CNI PodInfoProvider")
+		}
+	default:
+		logger.Printf("Initializing from Kubernetes")
+		podInfoByIPProvider = cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) {
+			pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{ //nolint:govet // ignore err shadow
+				FieldSelector: "spec.nodeName=" + nodeName,
+			})
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to list Pods for PodInfoProvider")
+			}
+			podInfo, err := cns.KubePodsToPodInfoByIP(pods.Items)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to convert Pods to PodInfoByIP")
+			}
+			return podInfo, nil
+		})
+	}
+
+	return podInfoByIPProvider, nil
+}
+
 // createOrUpdateNodeInfoCRD polls imds to learn the VM Unique ID and then creates or updates the NodeInfo CRD
 // with that vm unique ID
 func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, node *corev1.Node) error {

hack/aks/Makefile

@@ -8,17 +8,18 @@ AZIMG   = mcr.microsoft.com/azure-cli
 AZCLI   ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az
 
 # overrideable defaults
-AUTOUPGRADE     ?= patch
-K8S_VER         ?= 1.30
-NODE_COUNT      ?= 2
-NODE_COUNT_WIN	?= $(NODE_COUNT)
-NODEUPGRADE     ?= NodeImage
-OS				?= linux # Used to signify if you want to bring up a windows nodePool on byocni clusters
-OS_SKU          ?= Ubuntu
-OS_SKU_WIN      ?= Windows2022
-REGION          ?= westus2
-VM_SIZE	        ?= Standard_B2s
-VM_SIZE_WIN     ?= Standard_B2s
+AUTOUPGRADE          ?= patch
+K8S_VER              ?= 1.30
+NODE_COUNT           ?= 2
+NODE_COUNT_WIN	     ?= $(NODE_COUNT)
+NODEUPGRADE          ?= NodeImage
+OS				     ?= linux # Used to signify if you want to bring up a windows nodePool on byocni clusters
+OS_SKU               ?= Ubuntu
+OS_SKU_WIN           ?= Windows2022
+REGION               ?= westus2
+VM_SIZE	             ?= Standard_B2s
+VM_SIZE_WIN          ?= Standard_B2s
+KUBE_PROXY_JSON_PATH ?= ./kube-proxy.json
 
 # overrideable variables
 SUB        ?= $(AZURE_SUBSCRIPTION)
@@ -103,13 +104,13 @@ nodesubnet-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an NodeSubne
 		--kubernetes-version $(K8S_VER) \
 		--node-count $(NODE_COUNT) \
 		--node-vm-size $(VM_SIZE) \
-		--load-balancer-sku basic \
+		--load-balancer-sku standard \
 		--max-pods 250 \
 		--network-plugin none \
 		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
 		--os-sku $(OS_SKU) \
 		--no-ssh-key \
-		--kube-proxy-config ./kube-proxy.json \
+		--kube-proxy-config $(KUBE_PROXY_JSON_PATH) \
 		--yes
 	@$(MAKE) set-kubeconf
 
@@ -146,7 +147,7 @@ overlay-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO
 		--pod-cidr 192.168.0.0/16 \
 		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
 		--no-ssh-key \
-		--kube-proxy-config ./kube-proxy.json \
+		--kube-proxy-config $(KUBE_PROXY_JSON_PATH) \
 		--yes
 	@$(MAKE) set-kubeconf
 
@@ -215,7 +216,7 @@ swift-byocni-nokubeproxy-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI clus
 		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
 		--no-ssh-key \
 		--os-sku $(OS_SKU) \
-		--kube-proxy-config ./kube-proxy.json \
+		--kube-proxy-config $(KUBE_PROXY_JSON_PATH) \
 		--yes
 	@$(MAKE) set-kubeconf
 
@@ -305,7 +306,7 @@ vnetscale-swift-byocni-nokubeproxy-up: rg-up vnetscale-swift-net-up ## Bring up
 		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
 		--no-ssh-key \
 		--os-sku $(OS_SKU) \
-		--kube-proxy-config ./kube-proxy.json \
+		--kube-proxy-config $(KUBE_PROXY_JSON_PATH) \
 		--yes
 	@$(MAKE) set-kubeconf
 
@@ -434,7 +435,7 @@ dualstack-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up a Dualstack o
 		--ip-families ipv4,ipv6 \
 		--aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \
 		--no-ssh-key \
-		--kube-proxy-config ./kube-proxy.json \
+		--kube-proxy-config $(KUBE_PROXY_JSON_PATH) \
 		--yes
 	@$(MAKE) set-kubeconf