CNS Pool Scaling (#668)

* init with test framework

* handle multiple batch size pool request

* test verification

* handle batch size multiples

* address feedback

* fix test

* remove test struct from fakes

* fix nil test

* fix tests

* feedback

* scale down

* init scale down

scale down testing

test work

reconcile fake requestcontroller

test revision

check for cached nonreleased ipconfigs

remove test struct from decrease fakes

initial integration

fix interface signature

test get pending release

futher integration tseting

test fixes

* start feedback

* update request controller

* fixed tests

* test fix

* ipampoolmonitor in cns not request controller

* init fake ipam pool monitor

Author: matmerr

cns/api.go

@@ -4,9 +4,11 @@
 package cns
 
 import (
+	"context"
 	"encoding/json"
 
 	"github.com/Azure/azure-container-networking/cns/common"
+	nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
 )
 
 // Container Network Service remote API Contract
@@ -36,8 +38,10 @@ type HTTPService interface {
 	SetNodeOrchestrator(*SetOrchestratorTypeRequest)
 	SyncNodeStatus(string, string, string, json.RawMessage) (int, string)
 	GetAvailableIPConfigs() []IPConfigurationStatus
+	GetAllocatedIPConfigs() []IPConfigurationStatus
+	GetPendingReleaseIPConfigs() []IPConfigurationStatus
 	GetPodIPConfigState() map[string]IPConfigurationStatus
-	MarkIPsAsPending(numberToMark int) (map[string]SecondaryIPConfig, error)
+	MarkIPsAsPending(numberToMark int) (map[string]IPConfigurationStatus, error)
 }
 
 // This is used for KubernetesCRD orchastrator Type where NC has multiple ips.
@@ -160,16 +164,9 @@ type NodeConfiguration struct {
 	NodeID     string
 	NodeSubnet Subnet
 }
-
 type IPAMPoolMonitor interface {
-	Start() error
-	UpdatePoolLimitsTransacted(ScalarUnits)
-}
-
-type ScalarUnits struct {
-	BatchSize               int64
-	RequestThresholdPercent int64
-	ReleaseThresholdPercent int64
+	Start(ctx context.Context, poolMonitorRefreshMilliseconds int) error
+	Update(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
 }
 
 // Response describes generic response from CNS.

cns/cnsclient/apiclient.go

@@ -1,9 +1,12 @@
 package cnsclient
 
-import "github.com/Azure/azure-container-networking/cns"
+import (
+	"github.com/Azure/azure-container-networking/cns"
+	nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
+)
 
 // APIClient interface to update cns state
 type APIClient interface {
-	ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalarUnits cns.ScalarUnits) error
-	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest, scalarUnits cns.ScalarUnits) error
+	ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
 }

cns/cnsclient/cnsclient_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/common"
 	"github.com/Azure/azure-container-networking/cns/fakes"
-	"github.com/Azure/azure-container-networking/cns/ipampoolmonitor"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/restserver"
 	"github.com/Azure/azure-container-networking/log"
@@ -31,6 +30,10 @@ const (
 	gatewayIp           = "10.0.0.1"
 	subnetPrfixLength   = 24
 	dockerContainerType = cns.Docker
+	releasePercent      = 50
+	requestPercent      = 100
+	batchSize           = 10
+	initPoolSize        = 10
 )
 
 var (
@@ -39,7 +42,6 @@ var (
 
 func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 	var ipConfig cns.IPConfiguration
-	var scalarUnits cns.ScalarUnits
 	ipConfig.DNSServers = dnsservers
 	ipConfig.GatewayIPAddress = gatewayIp
 	var ipSubnet cns.IPSubnet
@@ -63,7 +65,7 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 		SecondaryIPConfigs:   secondaryIPConfigs,
 	}
 
-	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, scalarUnits)
+	returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize))
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
@@ -123,7 +125,7 @@ func TestMain(m *testing.M) {
 	httpRestService, err := restserver.NewHTTPRestService(&config, fakes.NewFakeImdsClient())
 	svc = httpRestService.(*restserver.HTTPRestService)
 	svc.Name = "cns-test-server"
-	svc.PoolMonitor = ipampoolmonitor.NewCNSIPAMPoolMonitor(nil, nil)
+	svc.IPAMPoolMonitor = fakes.NewIPAMPoolMonitorFake()
 
 	if err != nil {
 		logger.Errorf("Failed to create CNS object, err:%v.\n", err)

cns/cnsclient/httpapi/client.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/restserver"
+	nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
 )
 
 // Client implements APIClient interface. Used to update CNS state
@@ -13,8 +14,8 @@ type Client struct {
 }
 
 // CreateOrUpdateNC updates cns state
-func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalarUnits cns.ScalarUnits) error {
-	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest, scalarUnits)
+func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+	returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest, scalar, spec)
 
 	if returnCode != 0 {
 		return fmt.Errorf("Failed to Create NC request: %+v, errorCode: %d", ncRequest, returnCode)
@@ -24,8 +25,8 @@ func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerReque
 }
 
 // ReconcileNCState initializes cns state
-func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalarUnits cns.ScalarUnits) error {
-	returnCode := client.RestService.ReconcileNCState(ncRequest, podInfoByIP, scalarUnits)
+func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error {
+	returnCode := client.RestService.ReconcileNCState(ncRequest, podInfoByIP, scalar, spec)
 
 	if returnCode != 0 {
 		return fmt.Errorf("Failed to Reconcile ncState: ncRequest %+v, podInfoMap: %+v, errorCode: %d", *ncRequest, podInfoByIP, returnCode)