feat: add multi-tenant NetworkContainer controller (#876)

Author: feiskyer

Makefile

@@ -51,6 +51,8 @@ CNSFILES = \
 	$(wildcard cns/networkcontainers/*.go) \
 	$(wildcard cns/requestcontroller/*.go) \
 	$(wildcard cns/requestcontroller/kubecontroller/*.go) \
+	$(wildcard cns/multitenantcontroller/*.go) \
+	$(wildcard cns/multitenantcontroller/multitenantoperator/*.go) \
 	$(wildcard cns/fakes/*.go) \
 	$(COREFILES) \
 	$(CNMFILES)
@@ -181,13 +183,13 @@ azure-npm: $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) npm-archive
 endif
 
 ifeq ($(GOOS),linux)
-all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm 
+all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm
 else
 all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns
 endif
 
 ifeq ($(GOOS),linux)
-all-images: azure-npm-image azure-cns-image 
+all-images: azure-npm-image azure-cns-image
 else
 all-images:
 	@echo "Nothing to build. Skip."
@@ -223,7 +225,7 @@ $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES)
 # Build the Azure CLI network plugin.
 $(ACNCLI_BUILD_DIR)/acncli$(EXE_EXT): $(CNIFILES)
 	CGO_ENABLED=0 go build -v -o $(ACNCLI_BUILD_DIR)/acn$(EXE_EXT) -ldflags "-X main.version=$(VERSION)" -gcflags="-dwarflocationlists=true" $(ACNCLI_DIR)/*.go
- 
+
 # Build the Azure CNS Service.
 $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES)
 	go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(cnsaipath)=$(CNS_AI_ID)" -gcflags="-dwarflocationlists=true" $(CNS_DIR)/*.go
@@ -259,7 +261,7 @@ all-containerized:
 
 # Make both linux and windows binaries
 .PHONY: all-binaries-platforms
-all-binaries-platforms: 
+all-binaries-platforms:
 	export GOOS=linux; make all-binaries
 	export GOOS=windows; make all-binaries
 
@@ -268,7 +270,7 @@ all-binaries-platforms:
 tools: acncli
 
 .PHONY: tools-images
-tools-images: 
+tools-images:
 	docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
 
 # Build the Azure CNM plugin image, installable with "docker plugin install".
@@ -409,7 +411,7 @@ ifeq ($(GOOS),linux)
 	cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR)
 	chmod 0755 $(CNI_SWIFT_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT)
 	cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
-endif	
+endif
 
 # Create a CNM archive for the target platform.
 .PHONY: cnm-archive
@@ -453,7 +455,7 @@ endif
 .PHONY: release
 release:
 	./scripts/semver-release.sh
-	
+
 
 PRETTYGOTEST := $(shell command -v gotest 2> /dev/null)
 

cns/NetworkContainerContract.go

@@ -69,9 +69,10 @@ const (
 
 // ChannelMode :- CNS channel modes
 const (
-	Direct  = "Direct"
-	Managed = "Managed"
-	CRD     = "CRD"
+	Direct         = "Direct"
+	Managed        = "Managed"
+	CRD            = "CRD"
+	MultiTenantCRD = "MultiTenantCRD"
 )
 
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.

cns/cnsclient/apiclient.go

@@ -8,5 +8,8 @@ import (
 // APIClient interface to update cns state
 type APIClient interface {
 	ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
-	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest) error
+	UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error)
+	DeleteNC(nc cns.DeleteNetworkContainerRequest) error
 }

cns/cnsclient/cnsclient_test.go

@@ -73,10 +73,15 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 		Version: "-1",
 	}
 
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req)
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
+
+	returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
+	if returnCode != 0 {
+		t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode)
+	}
 }
 
 func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) {

cns/cnsclient/httpapi/client.go

@@ -14,8 +14,8 @@ type Client struct {
 }
 
 // CreateOrUpdateNC updates cns state
-func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
-	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest, scalar, spec)
+func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest) error {
+	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest)
 
 	if returnCode != 0 {
 		return fmt.Errorf("Failed to Create NC request: %+v, errorCode: %d", ncRequest, returnCode)
@@ -24,6 +24,17 @@ func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerReque
 	return nil
 }
 
+// UpdateIPAMPoolMonitor updates IPAM pool monitor.
+func (client *Client) UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+	returnCode := client.RestService.UpdateIPAMPoolMonitorInternal(scalar, spec)
+
+	if returnCode != 0 {
+		return fmt.Errorf("Failed to update IPAM pool monitor scalar: %+v, spec: %+v, errorCode: %d", scalar, spec, returnCode)
+	}
+
+	return nil
+}
+
 // ReconcileNCState initializes cns state
 func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
 	returnCode := client.RestService.ReconcileNCState(ncRequest, podInfoByIP, scalar, spec)
@@ -34,3 +45,24 @@ func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequ
 
 	return nil
 }
+
+func (client *Client) GetNC(req cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) {
+	response, returnCode := client.RestService.GetNetworkContainerInternal(req)
+	if returnCode != 0 {
+		if returnCode == restserver.UnknownContainerID {
+			return response, fmt.Errorf("NotFound")
+		}
+		return response, fmt.Errorf("Failed to get NC, request: %+v, errorCode: %d", req, returnCode)
+	}
+
+	return response, nil
+}
+
+func (client *Client) DeleteNC(req cns.DeleteNetworkContainerRequest) error {
+	returnCode := client.RestService.DeleteNetworkContainerInternal(req)
+	if returnCode != 0 {
+		return fmt.Errorf("Failed to delete NC, request: %+v, errorCode: %d", req, returnCode)
+	}
+
+	return nil
+}

cns/multitenantcontroller/mockclients/README.md

@@ -0,0 +1,8 @@
+# Mock Clients
+
+Run the following command to generate mock clients:
+
+```sh
+mockgen -source=$GOPATH/src/github.com/Azure/azure-container-networking/cns/cnsclient/apiclient.go -package=mockclients APIClient >cnsclient.go
+mockgen -source=$GOPATH/src/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go -package=mockclients Client >kubeclient.go
+```