feat: support for cilium + nodesubnet

.pipelines/pipeline.yaml

@@ -261,6 +261,17 @@ stages:
         k8sVersion: ""
         dependsOn: "containerize"
 
+    # Cilium Nodesubnet E2E tests
+    - template: singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml
+      parameters:
+        name: "cilium_nodesubnet_e2e"
+        displayName: Cilium NodeSubnet
+        clusterType: nodesubnet-byocni-nokubeproxy-up
+        clusterName: "cilndsubnete2e"
+        vmSize: Standard_B2s
+        k8sVersion: ""
+        dependsOn: "containerize"
+
     # Cilium Overlay E2E tests
     - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
       parameters:
@@ -405,6 +416,7 @@ stages:
         - azure_overlay_stateless_e2e
         - aks_swift_e2e
         - cilium_e2e
+        - cilium_nodesubnet_e2e
         - cilium_overlay_e2e
         - cilium_h_overlay_e2e
         - aks_ubuntu_22_linux_e2e
@@ -425,6 +437,10 @@ stages:
               cilium_e2e:
                 name: cilium_e2e
                 clusterName: "ciliume2e"
+                region: $(REGION_AKS_CLUSTER_TEST)                
+              cilium_nodesubnet_e2e:
+                name: cilium_nodesubnet_e2e
+                clusterName: "cilndsubnete2e"
                 region: $(REGION_AKS_CLUSTER_TEST)
               cilium_overlay_e2e:
                 name: cilium_overlay_e2e

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml

@@ -71,9 +71,9 @@ stages:
           os: ${{ parameters.os }}
           datapath: true
           dns: true
+          cni: cilium
           portforward: true
           service: true
-          hostport: true
           dependsOn: ${{ parameters.name }}
 
       - job: failedE2ELogs

.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml

@@ -57,6 +57,7 @@ steps:
         pwd
         kubectl cluster-info
         kubectl get po -owide -A
+        CILIUM_VERSION_TAG=${CILIUM_NODE_SUBNET_VERSION}
         echo "install Cilium ${CILIUM_VERSION_TAG}"
         export DIR=${CILIUM_VERSION_TAG%.*}
         echo "installing files from ${DIR}"

cns/fakes/nmagentclientfake.go

@@ -14,9 +14,10 @@ import (
 
 // NMAgentClientFake can be used to query to VM Host info.
 type NMAgentClientFake struct {
-	SupportedAPIsF    func(context.Context) ([]string, error)
-	GetNCVersionListF func(context.Context) (nmagent.NCVersionList, error)
-	GetHomeAzF        func(context.Context) (nmagent.AzResponse, error)
+	SupportedAPIsF      func(context.Context) ([]string, error)
+	GetNCVersionListF   func(context.Context) (nmagent.NCVersionList, error)
+	GetHomeAzF          func(context.Context) (nmagent.AzResponse, error)
+	GetInterfaceIPInfoF func(ctx context.Context) (nmagent.Interfaces, error)
 }
 
 func (n *NMAgentClientFake) SupportedAPIs(ctx context.Context) ([]string, error) {
@@ -30,3 +31,7 @@ func (n *NMAgentClientFake) GetNCVersionList(ctx context.Context) (nmagent.NCVer
 func (n *NMAgentClientFake) GetHomeAz(ctx context.Context) (nmagent.AzResponse, error) {
 	return n.GetHomeAzF(ctx)
 }
+
+func (n *NMAgentClientFake) GetInterfaceIPInfo(ctx context.Context) (nmagent.Interfaces, error) {
+	return n.GetInterfaceIPInfoF(ctx)
+}

cns/restserver/helper_for_nodesubnet_test.go

@@ -0,0 +1,83 @@
+package restserver
+
+import (
+	"context"
+	"net/netip"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/common"
+	"github.com/Azure/azure-container-networking/cns/fakes"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	acn "github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/nmagent"
+	"github.com/Azure/azure-container-networking/store"
+)
+
+func GetRestServiceObjectForNodeSubnetTest(t *testing.T, generator CNIConflistGenerator) *HTTPRestService {
+	config := &common.ServiceConfig{
+		Name:        "test",
+		Version:     "1.0",
+		ChannelMode: "AzureHost",
+		Store:       store.NewMockStore("test"),
+	}
+	interfaces := nmagent.Interfaces{
+		Entries: []nmagent.Interface{
+			{
+				MacAddress: nmagent.MACAddress{0x00, 0x0D, 0x3A, 0xF9, 0xDC, 0xA6},
+				IsPrimary:  true,
+				InterfaceSubnets: []nmagent.InterfaceSubnet{
+					{
+						Prefix: "10.240.0.0/16",
+						IPAddress: []nmagent.NodeIP{
+							{
+								Address:   nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 240, 0, 5})),
+								IsPrimary: true,
+							},
+							{
+								Address:   nmagent.IPAddress(netip.AddrFrom4([4]byte{10, 240, 0, 6})),
+								IsPrimary: false,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	svc, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store)
+	if err != nil {
+		return nil
+	}
+
+	svc.SetOption(acn.OptCnsURL, "")
+	svc.SetOption(acn.OptCnsPort, "")
+	err = svc.Initialize(config)
+	if err != nil {
+		return nil
+	}
+
+	t.Cleanup(func() { svc.Uninitialize() })
+
+	return &HTTPRestService{
+		Service:              svc,
+		cniConflistGenerator: generator,
+		state:                &httpRestServiceState{},
+		PodIPConfigState:     make(map[string]cns.IPConfigurationStatus),
+		nma: &fakes.NMAgentClientFake{
+			GetInterfaceIPInfoF: func(_ context.Context) (nmagent.Interfaces, error) {
+				return interfaces, nil
+			},
+		},
+	}
+}
+
+// SetCNIConflistGenerator sets the CNIConflistGenerator for the HTTPRestService.
+func (service *HTTPRestService) SetCNIConflistGenerator(generator CNIConflistGenerator) {
+	service.cniConflistGenerator = generator
+}
+
+// GetNodesubnetIPFetcher gets the nodesubnet.IPFetcher from the HTTPRestService.
+func (service *HTTPRestService) GetNodesubnetIPFetcher() *nodesubnet.IPFetcher {
+	return service.nodesubnetIPFetcher
+}

cns/restserver/nodesubnet.go

@@ -0,0 +1,58 @@
+package restserver
+
+import (
+	"context"
+	"net/netip"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	nodesubnet "github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+)
+
+var _ nodesubnet.IPConsumer = &HTTPRestService{}
+
+// Implement the UpdateIPsForNodeSubnet method for HTTPRestService
+func (service *HTTPRestService) UpdateIPsForNodeSubnet(secondaryIPs []netip.Addr) error {
+	secondaryIPStrs := make([]string, len(secondaryIPs))
+	for i, ip := range secondaryIPs {
+		secondaryIPStrs[i] = ip.String()
+	}
+
+	networkContainerRequest := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPStrs)
+
+	code, msg := service.saveNetworkContainerGoalState(*networkContainerRequest)
+	if code != types.Success {
+		return errors.Errorf("failed to save fetched ips. code: %d, message %s", code, msg)
+	}
+
+	logger.Debugf("IP change processed successfully")
+
+	// saved NC successfully, generate conflist to indicate CNS is ready
+	go service.MustGenerateCNIConflistOnce()
+	return nil
+}
+
+func (service *HTTPRestService) InitializeNodeSubnet(ctx context.Context, podInfoByIPProvider cns.PodInfoByIPProvider) error {
+	// Set orchestrator type
+	orchestrator := cns.SetOrchestratorTypeRequest{
+		OrchestratorType: cns.KubernetesCRD,
+	}
+	service.SetNodeOrchestrator(&orchestrator)
+
+	if podInfoByIPProvider == nil {
+		logger.Printf("PodInfoByIPProvider is nil, this usually means no saved endpoint state. Skipping reconciliation")
+	} else if _, err := nodesubnet.ReconcileInitialCNSState(ctx, service, podInfoByIPProvider); err != nil {
+		return errors.Wrap(err, "reconcile initial CNS state")
+	}
+	// statefile (if any) is reconciled. Initialize the IP fetcher. Start the IP fetcher only after the service is started,
+	// and any pending async delete operations are completed.
+	service.nodesubnetIPFetcher = nodesubnet.NewIPFetcher(service.nma, service, 0, 0, logger.Log)
+
+	return nil
+}
+
+func (service *HTTPRestService) StartNodeSubnet(ctx context.Context) {
+	service.nodesubnetIPFetcher.Start(ctx)
+}

cns/restserver/nodesubnet_test.go

@@ -0,0 +1,76 @@
+package restserver_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+)
+
+// Mock implementation of PodInfoByIPProvider
+type MockPodInfoByIPProvider struct{}
+
+func (m *MockPodInfoByIPProvider) PodInfoByIP() (res map[string]cns.PodInfo, err error) {
+	return res, nil
+}
+
+// Mock implementation of CNIConflistGenerator
+type MockCNIConflistGenerator struct {
+	GenerateCalled chan struct{}
+}
+
+func (m *MockCNIConflistGenerator) Generate() error {
+	close(m.GenerateCalled)
+	return nil
+}
+
+func (m *MockCNIConflistGenerator) Close() error {
+	// Implement the Close method logic here if needed
+	return nil
+}
+
+func TestNodeSubnet(t *testing.T) {
+	mockPodInfoProvider := &MockPodInfoByIPProvider{}
+
+	// Create a real HTTPRestService object
+	mockCNIConflistGenerator := &MockCNIConflistGenerator{
+		GenerateCalled: make(chan struct{}),
+	}
+	service := restserver.GetRestServiceObjectForNodeSubnetTest(t, mockCNIConflistGenerator)
+	ctx, cancel := testContext(t)
+	defer cancel()
+
+	err := service.InitializeNodeSubnet(ctx, mockPodInfoProvider)
+	service.StartNodeSubnet(ctx)
+
+	if service.GetNodesubnetIPFetcher() == nil {
+		t.Error("NodeSubnetIPFetcher is not initialized")
+	}
+
+	if err != nil {
+		t.Fatalf("InitializeNodeSubnet returned an error: %v", err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Errorf("test context done - %s", ctx.Err())
+		return
+	case <-mockCNIConflistGenerator.GenerateCalled:
+		break
+	}
+}
+
+// testContext creates a context from the provided testing.T that will be
+// canceled if the test suite is terminated.
+func testContext(t *testing.T) (context.Context, context.CancelFunc) {
+	if deadline, ok := t.Deadline(); ok {
+		return context.WithDeadline(context.Background(), deadline)
+	}
+	return context.WithCancel(context.Background())
+}
+
+func init() {
+	logger.InitLogger("testlogs", 0, 0, "./")
+}

cns/restserver/restserver.go

@@ -13,6 +13,7 @@ import (
 	"github.com/Azure/azure-container-networking/cns/dockerclient"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/networkcontainers"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
 	"github.com/Azure/azure-container-networking/cns/routes"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/cns/types/bounded"
@@ -40,6 +41,7 @@ type nmagentClient interface {
 	SupportedAPIs(context.Context) ([]string, error)
 	GetNCVersionList(context.Context) (nma.NCVersionList, error)
 	GetHomeAz(context.Context) (nma.AzResponse, error)
+	GetInterfaceIPInfo(ctx context.Context) (nma.Interfaces, error)
 }
 
 type wireserverProxy interface {
@@ -76,6 +78,7 @@ type HTTPRestService struct {
 	IPConfigsHandlerMiddleware cns.IPConfigsHandlerMiddleware
 	PnpIDByMacAddress          map[string]string
 	imdsClient                 imdsClient
+	nodesubnetIPFetcher        *nodesubnet.IPFetcher
 }
 
 type CNIConflistGenerator interface {