Merge pull request #825 from kmurudi/CNS_Debug_HTTPRestService_Contexts

[CNS] Debug - Expose In-memory data from HTTPRestService

Author: kmurudi

cns/NetworkContainerContract.go

@@ -24,6 +24,7 @@ const (
 	ReleaseIPConfig                          = "/network/releaseipconfig"
 	GetIPAddresses                           = "/debug/getipaddresses"
 	GetPodIPOrchestratorContext              = "/debug/getpodcontext"
+	GetHTTPRestData                          = "/debug/getrestdata"
 )
 
 // NetworkContainer Prefixes

cns/api.go

@@ -175,6 +175,15 @@ type NodeConfiguration struct {
 type IPAMPoolMonitor interface {
 	Start(ctx context.Context, poolMonitorRefreshMilliseconds int) error
 	Update(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error
+	GetStateSnapshot() IpamPoolMonitorStateSnapshot
+}
+
+//struct to expose state values for IPAMPoolMonitor struct
+type IpamPoolMonitorStateSnapshot struct {
+	MinimumFreeIps           int64
+	MaximumFreeIps           int64
+	UpdatingIpsNotInUseCount int
+	CachedNNC                nnc.NodeNetworkConfig
 }
 
 // Response describes generic response from CNS.

cns/cnsclient/cli.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/restserver"
 )
 
 const (
@@ -16,6 +17,7 @@ const (
 	getAllArg            = "All"
 	getPendingReleaseArg = "PendingRelease"
 	getPodCmdArg         = "getPodContexts"
+	getInMemoryData      = "getInMemory"
 
 	releaseArg = "release"
 
@@ -30,6 +32,7 @@ var (
 	availableCmds = []string{
 		getCmdArg,
 		getPodCmdArg,
+		getInMemoryData,
 	}
 
 	getFlags = []string{
@@ -53,6 +56,8 @@ func HandleCNSClientCommands(cmd, arg string) error {
 		return getCmd(cnsClient, arg)
 	case strings.EqualFold(getPodCmdArg, cmd):
 		return getPodCmd(cnsClient)
+	case strings.EqualFold(getInMemoryData, cmd):
+		return getInMemory(cnsClient)
 	default:
 		return fmt.Errorf("No debug cmd supplied, options are: %v", getCmdArg)
 	}
@@ -119,3 +124,20 @@ func printPodContext(podContext map[string]string) {
 		i++
 	}
 }
+
+func getInMemory(client *CNSClient) error {
+
+	inmemoryData, err := client.GetHTTPServiceData()
+	if err != nil {
+		return err
+	}
+
+	printInMemoryStruct(inmemoryData.HttpRestServiceData)
+	return nil
+}
+
+func printInMemoryStruct(data restserver.HttpRestServiceData) {
+	fmt.Println("PodIPIDByOrchestratorContext: ", data.PodIPIDByOrchestratorContext)
+	fmt.Println("PodIPConfigState: ", data.PodIPConfigState)
+	fmt.Println("IPAMPoolMonitor: ", data.IPAMPoolMonitor)
+}

cns/cnsclient/cnsclient.go

@@ -410,3 +410,42 @@ func (cnsClient *CNSClient) GetPodOrchestratorContext() (map[string]string, erro
 
 	return resp.PodContext, err
 }
+
+//GetHTTPServiceData gets all public in-memory struct details for debugging purpose
+func (cnsClient *CNSClient) GetHTTPServiceData() (restserver.GetHTTPServiceDataResponse, error) {
+	var (
+		resp restserver.GetHTTPServiceDataResponse
+		err  error
+		res  *http.Response
+	)
+
+	url := cnsClient.connectionURL + cns.GetHTTPRestData
+	log.Printf("GetHTTPServiceStruct url %v", url)
+
+	res, err = http.Get(url)
+	if err != nil {
+		log.Errorf("[Azure CNSClient] HTTP Get returned error %v", err.Error())
+		return resp, err
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("[Azure CNSClient] GetHTTPServiceStruct invalid http status code: %v", res.StatusCode)
+		log.Errorf(errMsg)
+		return resp, fmt.Errorf(errMsg)
+	}
+
+	err = json.NewDecoder(res.Body).Decode(&resp)
+	if err != nil {
+		log.Errorf("[Azure CNSClient] Error received while parsing GetHTTPServiceStruct response resp:%v err:%v", res.Body, err.Error())
+		return resp, err
+	}
+
+	if resp.Response.ReturnCode != 0 {
+		log.Errorf("[Azure CNSClient] GetTTPServiceStruct received error response :%v", resp.Response.Message)
+		return resp, fmt.Errorf(resp.Response.Message)
+	}
+
+	return resp, err
+}

cns/cnsclient/cnsclient_test.go

@@ -12,6 +12,9 @@ import (
 	"strconv"
 	"testing"
 
+	nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/common"
 	"github.com/Azure/azure-container-networking/cns/fakes"
@@ -129,7 +132,38 @@ func TestMain(m *testing.M) {
 	httpRestService, err := restserver.NewHTTPRestService(&config, fakes.NewFakeImdsClient(), fakes.NewFakeNMAgentClient())
 	svc = httpRestService.(*restserver.HTTPRestService)
 	svc.Name = "cns-test-server"
-	svc.IPAMPoolMonitor = fakes.NewIPAMPoolMonitorFake()
+	fakeNNC := nnc.NodeNetworkConfig{
+		TypeMeta:   metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{},
+		Spec: nnc.NodeNetworkConfigSpec{
+			RequestedIPCount: 16,
+			IPsNotInUse:      []string{"abc"},
+		},
+		Status: nnc.NodeNetworkConfigStatus{
+			Scaler: nnc.Scaler{
+				BatchSize:               10,
+				ReleaseThresholdPercent: 50,
+				RequestThresholdPercent: 40,
+			},
+			NetworkContainers: []nnc.NetworkContainer{
+				{
+					ID:         "nc1",
+					PrimaryIP:  "10.0.0.11",
+					SubnetName: "sub1",
+					IPAssignments: []nnc.IPAssignment{
+						{
+							Name: "ip1",
+							IP:   "10.0.0.10",
+						},
+					},
+					DefaultGateway:     "10.0.0.1",
+					SubnetAddressSpace: "10.0.0.0/24",
+					Version:            2,
+				},
+			},
+		},
+	}
+	svc.IPAMPoolMonitor = &fakes.IPAMPoolMonitorFake{FakeMinimumIps: 10, FakeMaximumIps: 20, FakeIpsNotInUseCount: 13, FakecachedNNC: fakeNNC}
 
 	if err != nil {
 		logger.Errorf("Failed to create CNS object, err:%v.\n", err)
@@ -254,8 +288,7 @@ func TestCNSClientPodContextApi(t *testing.T) {
 	podNamespace := "testpodnamespace"
 	desiredIpAddress := "10.0.0.5"
 
-	secondaryIps := make([]string, 0)
-	secondaryIps = append(secondaryIps, desiredIpAddress)
+	secondaryIps := []string{desiredIpAddress}
 	cnsClient, _ := InitCnsClient("")
 
 	addTestStateToRestServer(t, secondaryIps)
@@ -282,4 +315,79 @@ func TestCNSClientPodContextApi(t *testing.T) {
 	}
 
 	t.Log(podcontext)
+
+	// release requested IP address, expect success
+	err = cnsClient.ReleaseIPAddress(orchestratorContext)
+	if err != nil {
+		t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err)
+	}
+}
+
+func TestCNSClientDebugAPI(t *testing.T) {
+	podName := "testpodname"
+	podNamespace := "testpodnamespace"
+	desiredIpAddress := "10.0.0.5"
+
+	secondaryIps := []string{desiredIpAddress}
+	cnsClient, _ := InitCnsClient("")
+
+	addTestStateToRestServer(t, secondaryIps)
+
+	podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace}
+	orchestratorContext, err := json.Marshal(podInfo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// request IP address
+	_, err1 := cnsClient.RequestIPAddress(orchestratorContext)
+	if err1 != nil {
+		t.Fatalf("get IP from CNS failed with %+v", err1)
+	}
+
+	//test for debug api/cmd to get inmemory data from HTTPRestService
+	inmemory, err := cnsClient.GetHTTPServiceData()
+	if err != nil {
+		t.Errorf("Get in-memory http REST Struct failed %+v", err)
+	}
+
+	if len(inmemory.HttpRestServiceData.PodIPIDByOrchestratorContext) < 1 {
+		t.Errorf("OrchestratorContext map is expected but not returned")
+	}
+
+	//testing Pod IP Configuration Status values set for test
+	podConfig := inmemory.HttpRestServiceData.PodIPConfigState
+	for _, v := range podConfig {
+		if v.IPAddress != "10.0.0.5" || v.State != "Allocated" || v.NCID != "testNcId1" {
+			t.Errorf("Not the expected set values for testing IPConfigurationStatus, %+v", podConfig)
+		}
+	}
+	if len(inmemory.HttpRestServiceData.PodIPConfigState) < 1 {
+		t.Errorf("PodIpConfigState with atleast 1 entry expected but not returned.")
+	}
+
+	testIpamPoolMonitor := inmemory.HttpRestServiceData.IPAMPoolMonitor
+	if testIpamPoolMonitor.MinimumFreeIps != 10 || testIpamPoolMonitor.MaximumFreeIps != 20 || testIpamPoolMonitor.UpdatingIpsNotInUseCount != 13 {
+		t.Errorf("IPAMPoolMonitor state is not reflecting the initial set values, %+v", testIpamPoolMonitor)
+	}
+
+	//check for cached NNC Spec struct values
+	if testIpamPoolMonitor.CachedNNC.Spec.RequestedIPCount != 16 || len(testIpamPoolMonitor.CachedNNC.Spec.IPsNotInUse) != 1 {
+		t.Errorf("IPAMPoolMonitor cached NNC Spec is not reflecting the initial set values, %+v", testIpamPoolMonitor.CachedNNC.Spec)
+	}
+
+	//check for cached NNC Status struct values
+	if testIpamPoolMonitor.CachedNNC.Status.Scaler.BatchSize != 10 || testIpamPoolMonitor.CachedNNC.Status.Scaler.ReleaseThresholdPercent != 50 || testIpamPoolMonitor.CachedNNC.Status.Scaler.RequestThresholdPercent != 40 {
+		t.Errorf("IPAMPoolMonitor cached NNC Status is not reflecting the initial set values, %+v", testIpamPoolMonitor.CachedNNC.Status.Scaler)
+	}
+
+	if len(testIpamPoolMonitor.CachedNNC.Status.NetworkContainers) != 1 {
+		t.Errorf("Expected only one Network Container in the list, %+v", testIpamPoolMonitor.CachedNNC.Status.NetworkContainers)
+	}
+
+	t.Logf("In-memory Data: ")
+	t.Logf("PodIPIDByOrchestratorContext: %+v", inmemory.HttpRestServiceData.PodIPIDByOrchestratorContext)
+	t.Logf("PodIPConfigState: %+v", inmemory.HttpRestServiceData.PodIPConfigState)
+	t.Logf("IPAMPoolMonitor: %+v", inmemory.HttpRestServiceData.IPAMPoolMonitor)
+
 }

cns/fakes/ipampoolmonitorfake.go

@@ -3,10 +3,17 @@ package fakes
 import (
 	"context"
 
+	"github.com/Azure/azure-container-networking/cns"
+
 	nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha"
 )
 
-type IPAMPoolMonitorFake struct{}
+type IPAMPoolMonitorFake struct {
+	FakeMinimumIps       int
+	FakeMaximumIps       int
+	FakeIpsNotInUseCount int
+	FakecachedNNC        nnc.NodeNetworkConfig
+}
 
 func NewIPAMPoolMonitorFake() *IPAMPoolMonitorFake {
 	return &IPAMPoolMonitorFake{}
@@ -23,3 +30,12 @@ func (ipm *IPAMPoolMonitorFake) Update(scalar nnc.Scaler, spec nnc.NodeNetworkCo
 func (ipm *IPAMPoolMonitorFake) Reconcile() error {
 	return nil
 }
+
+func (ipm *IPAMPoolMonitorFake) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot {
+	return cns.IpamPoolMonitorStateSnapshot{
+		MinimumFreeIps:           int64(ipm.FakeMinimumIps),
+		MaximumFreeIps:           int64(ipm.FakeMaximumIps),
+		UpdatingIpsNotInUseCount: ipm.FakeIpsNotInUseCount,
+		CachedNNC:                ipm.FakecachedNNC,
+	}
+}

cns/ipampoolmonitor/ipampoolmonitor.go

@@ -15,9 +15,9 @@ import (
 type CNSIPAMPoolMonitor struct {
 	pendingRelease bool
 
-	cachedNNC   nnc.NodeNetworkConfig
+	cachedNNC                nnc.NodeNetworkConfig
 	updatingIpsNotInUseCount int
-	scalarUnits nnc.Scaler
+	scalarUnits              nnc.Scaler
 
 	httpService    cns.HTTPService
 	rc             requestcontroller.RequestController
@@ -30,9 +30,9 @@ type CNSIPAMPoolMonitor struct {
 func NewCNSIPAMPoolMonitor(httpService cns.HTTPService, rc requestcontroller.RequestController) *CNSIPAMPoolMonitor {
 	logger.Printf("NewCNSIPAMPoolMonitor: Create IPAM Pool Monitor")
 	return &CNSIPAMPoolMonitor{
-		pendingRelease:				false,
-		httpService:        		httpService,
-		rc:             			rc,
+		pendingRelease: false,
+		httpService:    httpService,
+		rc:             rc,
 	}
 }
 
@@ -71,7 +71,6 @@ func (pm *CNSIPAMPoolMonitor) Reconcile() error {
 	pendingReleaseIPCount := len(pm.httpService.GetPendingReleaseIPConfigs())
 	availableIPConfigCount := len(pm.httpService.GetAvailableIPConfigs()) // TODO: add pending allocation count to real cns
 	freeIPConfigCount := pm.cachedNNC.Spec.RequestedIPCount - int64(allocatedPodIPCount)
-
 	msg := fmt.Sprintf("[ipam-pool-monitor] Pool Size: %v, Goal Size: %v, BatchSize: %v, MinFree: %v, MaxFree:%v, Allocated: %v, Available: %v, Pending Release: %v, Free: %v, Pending Program: %v",
 		cnsPodIPConfigCount, pm.cachedNNC.Spec.RequestedIPCount, pm.scalarUnits.BatchSize, pm.MinimumFreeIps, pm.MaximumFreeIps, allocatedPodIPCount, availableIPConfigCount, pendingReleaseIPCount, freeIPConfigCount, pendingProgramCount)
 
@@ -202,7 +201,6 @@ func (pm *CNSIPAMPoolMonitor) cleanPendingRelease() error {
 
 	logger.Printf("[ipam-pool-monitor] cleanPendingRelease: UpdateCRDSpec succeeded for spec %+v", tempNNCSpec)
 
-
 	// save the updated state to cachedSpec
 	pm.cachedNNC.Spec = tempNNCSpec
 	pm.pendingRelease = false
@@ -249,3 +247,16 @@ func (pm *CNSIPAMPoolMonitor) Update(scalar nnc.Scaler, spec nnc.NodeNetworkConf
 
 	return nil
 }
+
+//this function sets the values for state in IPAMPoolMonitor Struct
+func (pm *CNSIPAMPoolMonitor) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot {
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
+
+	return cns.IpamPoolMonitorStateSnapshot{
+		MinimumFreeIps:           pm.MinimumFreeIps,
+		MaximumFreeIps:           pm.MaximumFreeIps,
+		UpdatingIpsNotInUseCount: pm.updatingIpsNotInUseCount,
+		CachedNNC:                pm.cachedNNC,
+	}
+}

cns/restserver/ipam.go

@@ -221,6 +221,38 @@ func (service *HTTPRestService) GetPodIPIDByOrchestratorContext() map[string]str
 	return service.PodIPIDByOrchestratorContext
 }
 
+func (service *HTTPRestService) GetHTTPRestDataHandler(w http.ResponseWriter, r *http.Request) {
+	var (
+		resp          GetHTTPServiceDataResponse
+		returnMessage string
+		err           error
+	)
+
+	defer func() {
+		if err != nil {
+			resp.Response.ReturnCode = UnexpectedError
+			resp.Response.Message = returnMessage
+		}
+
+		err = service.Listener.Encode(w, &resp)
+		logger.Response(service.Name, resp, resp.Response.ReturnCode, ReturnCodeToString(resp.Response.ReturnCode), err)
+	}()
+
+	resp.HttpRestServiceData = service.GetHTTPStruct()
+	return
+}
+
+func (service *HTTPRestService) GetHTTPStruct() HttpRestServiceData {
+	service.RLock()
+	defer service.RUnlock()
+
+	return HttpRestServiceData{
+		PodIPIDByOrchestratorContext: service.PodIPIDByOrchestratorContext,
+		PodIPConfigState:             service.PodIPConfigState,
+		IPAMPoolMonitor:              service.IPAMPoolMonitor.GetStateSnapshot(),
+	}
+}
+
 // GetPendingProgramIPConfigs returns list of IPs which are in pending program status
 func (service *HTTPRestService) GetPendingProgramIPConfigs() []cns.IPConfigurationStatus {
 	service.RLock()