CNS to DNC communication in Managed DNC (#574)

* initial changes for CNS->DNC support

* Adding changes for CNS to be compatible with managed DNC (reverse communication channel)

* adding NC version validation with respective NMA

* return errors for respective NC based on orchestrator context from CNI

* add nc version check via NMA

* adding logic to SyncNodeStatus and check if NCWaitingForUpdate for CniADD and CnsAttach calls

* addressing most of ashvin's comments

* adding managed config

* fat rebase

* addressing some comments

* slight optimizations...

* adding channel mode instead of managed bool

* set err in register node so that we keep looping

* addressing ashvin's comments

* fix test

* removing swift prefix mods for mdnc

* addressing tamanoha's comments

Co-authored-by: Jaeryn <[email protected]>

Author: jaer-tsun

cns/NetworkContainerContract.go

@@ -61,6 +61,13 @@ const (
 	PendingRelease = "PendingRelease"
 )
 
+// ChannelMode :- CNS channel modes
+const (
+	Direct  = "Direct"
+	Managed = "Managed"
+	CRD     = "CRD"
+)
+
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.
 type CreateNetworkContainerRequest struct {
 	Version                    string
@@ -318,3 +325,9 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate()
 	}
 	return nil
 }
+
+// NodeInfoResponse - Struct to hold the node info response.
+type NodeInfoResponse struct {
+	NetworkContainers  []CreateNetworkContainerRequest
+	GetNCVersionURLFmt string
+}

cns/common/service.go

@@ -13,11 +13,12 @@ import (
 
 // Service implements behavior common to all services.
 type Service struct {
-	Name    string
-	Version string
-	Options map[string]interface{}
-	ErrChan chan error
-	Store   store.KeyValueStore
+	Name        string
+	Version     string
+	Options     map[string]interface{}
+	ErrChan     chan error
+	Store       store.KeyValueStore
+	ChannelMode string
 }
 
 // ServiceAPI defines base interface.
@@ -30,25 +31,27 @@ type ServiceAPI interface {
 
 // ServiceConfig specifies common configuration.
 type ServiceConfig struct {
-	Name     string
-	Version  string
-	Listener *acn.Listener
-	ErrChan  chan error
-	Store    store.KeyValueStore
+	Name        string
+	Version     string
+	Listener    *acn.Listener
+	ErrChan     chan error
+	Store       store.KeyValueStore
+	ChannelMode string
 }
 
 // NewService creates a new Service object.
-func NewService(name, version string, store store.KeyValueStore) (*Service, error) {
+func NewService(name, version, channelMode string, store store.KeyValueStore) (*Service, error) {
 	logger.Debugf("[Azure CNS] Going to create a service object with name: %v. version: %v.", name, version)
 
 	svc := &Service{
-		Name:    name,
-		Version: version,
-		Options: make(map[string]interface{}),
-		Store:   store,
+		Name:        name,
+		Version:     version,
+		ChannelMode: channelMode,
+		Options:     make(map[string]interface{}),
+		Store:       store,
 	}
 
-	logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v.", name, version)
+	logger.Debugf("[Azure CNS] Finished creating service object with name: %v. version: %v. managed: %s", name, version, channelMode)
 	return svc, nil
 }
 
@@ -65,6 +68,7 @@ func (service *Service) Initialize(config *ServiceConfig) error {
 	service.ErrChan = config.ErrChan
 	service.Store = config.Store
 	service.Version = config.Version
+	service.ChannelMode = config.ChannelMode
 
 	logger.Debugf("[Azure CNS] nitialized service: %+v with config: %+v.", service, config)
 

cns/configuration/cns_config.json

@@ -7,5 +7,12 @@
         "HeartBeatIntervalInMins": 30,
         "DebugMode": false,
         "SnapshotIntervalInMins": 60
-    }
+    },
+    "ManagedSettings": {
+        "PrivateEndpoint": "",
+        "InfrastructureNetworkID": "",
+        "NodeID": "",
+        "NodeSyncIntervalInSeconds": 30
+    },
+    "ChannelMode": "Direct"
 }

cns/configuration/configuration.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/common"
 )
@@ -17,6 +18,8 @@ const (
 
 type CNSConfig struct {
 	TelemetrySettings TelemetrySettings
+	ManagedSettings   ManagedSettings
+	ChannelMode       string
 }
 
 type TelemetrySettings struct {
@@ -44,6 +47,13 @@ type TelemetrySettings struct {
 	SnapshotIntervalInMins int
 }
 
+type ManagedSettings struct {
+	PrivateEndpoint           string
+	InfrastructureNetworkID   string
+	NodeID                    string
+	NodeSyncIntervalInSeconds int
+}
+
 // This functions reads cns config file and save it in a structure
 func ReadConfig() (CNSConfig, error) {
 	var cnsConfig CNSConfig
@@ -99,7 +109,18 @@ func setTelemetrySettingDefaults(telemetrySettings *TelemetrySettings) {
 	}
 }
 
+// set managed setting defaults
+func setManagedSettingDefaults(managedSettings *ManagedSettings) {
+	if managedSettings.NodeSyncIntervalInSeconds == 0 {
+		managedSettings.NodeSyncIntervalInSeconds = 30
+	}
+}
+
 // Set Default values of CNS config if not specified
 func SetCNSConfigDefaults(config *CNSConfig) {
 	setTelemetrySettingDefaults(&config.TelemetrySettings)
+	setManagedSettingDefaults(&config.ManagedSettings)
+	if config.ChannelMode == "" {
+		config.ChannelMode = cns.Direct
+	}
 }

cns/dockerclient/dockerclient.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/cns/imdsclient"
 	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/platform"
 )
 
@@ -117,7 +118,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd
 
 	res, err := http.Post(
 		dockerClient.connectionURL+createNetworkPath,
-		"application/json; charset=utf-8",
+		common.JsonContent,
 		netConfigJSON)
 
 	if err != nil {
@@ -160,7 +161,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error {
 		return err
 	}
 
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set(common.ContentType, common.JsonContent)
 	client := &http.Client{}
 	res, err := client.Do(req)
 	if err != nil {

cns/nmagentclient/nmagentclient.go

@@ -9,6 +9,17 @@ import (
 	"github.com/Azure/azure-container-networking/common"
 )
 
+const (
+	WireserverIP = "168.63.129.16"
+)
+
+// NMANetworkContainerResponse - NMAgent response.
+type NMANetworkContainerResponse struct {
+	ResponseCode       string `json:"httpStatusCode"`
+	NetworkContainerID string `json:"networkContainerId"`
+	Version            string `json:"version"`
+}
+
 // JoinNetwork joins the given network
 func JoinNetwork(
 	networkID string,
@@ -62,3 +73,16 @@ func UnpublishNetworkContainer(
 
 	return response, err
 }
+
+// GetNetworkContainerVersion :- Retrieves NC version from NMAgent
+func GetNetworkContainerVersion(
+	networkContainerID,
+	getNetworkContainerVersionURL string) (*http.Response, error) {
+	logger.Printf("[NMAgentClient] GetNetworkContainerVersion NC: %s", networkContainerID)
+
+	response, err := common.GetHttpClient().Get(getNetworkContainerVersionURL)
+
+	logger.Printf("[NMAgentClient][Response] GetNetworkContainerVersion NC: %s. Response: %+v. Error: %v",
+		networkContainerID, response, err)
+	return response, err
+}

cns/restserver/api.go

@@ -824,7 +824,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr
 	err = service.Listener.Encode(w, &reserveResp)
 
 	// If the NC was created successfully, log NC snapshot.
-	if returnCode == 0 {
+	if returnCode == Success {
 		logNCSnapshot(req)
 	}
 
@@ -972,8 +972,6 @@ func (service *HTTPRestService) getNetworkContainerStatus(w http.ResponseWriter,
 	containerInfo := service.state.ContainerStatus
 	if containerInfo != nil {
 		containerDetails, ok = containerInfo[req.NetworkContainerid]
-	} else {
-		ok = false
 	}
 
 	var hostVersion string

cns/restserver/api_test.go

@@ -79,13 +79,13 @@ const (
 )
 
 func getInterfaceInfo(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/xml")
+	w.Header().Set(acncommon.ContentType, "application/xml")
 	output, _ := xml.Marshal(hostQueryResponse)
 	w.Write(output)
 }
 
 func nmagentHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+	w.Header().Set(acncommon.ContentType, acncommon.JsonContent)
 	w.WriteHeader(http.StatusOK)
 
 	if strings.Contains(r.RequestURI, "networkContainers") {
@@ -248,7 +248,7 @@ func TestGetNetworkContainerByOrchestratorContext(t *testing.T) {
 	setEnv(t)
 	setOrchestratorType(t, cns.Kubernetes)
 
-	err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "AzureContainerInstance")
+	err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance)
 	if err != nil {
 		t.Errorf("creatOrUpdateNetworkContainerWithName failed Err:%+v", err)
 		t.Fatal(err)
@@ -283,7 +283,7 @@ func TestGetNetworkContainerStatus(t *testing.T) {
 	setEnv(t)
 	setOrchestratorType(t, cns.Kubernetes)
 
-	err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps")
+	err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", cns.AzureContainerInstance)
 	if err != nil {
 		t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err)
 		t.Fatal(err)
@@ -597,7 +597,7 @@ func getNetworkContainerStatus(t *testing.T, name string) error {
 	var resp cns.GetNetworkContainerStatusResponse
 
 	getReq := &cns.GetNetworkContainerStatusRequest{
-		NetworkContainerid: "ethWebApp",
+		NetworkContainerid: name,
 	}
 
 	json.NewEncoder(&body).Encode(getReq)
@@ -624,7 +624,7 @@ func getInterfaceForContainer(t *testing.T, name string) error {
 	var resp cns.GetInterfaceForContainerResponse
 
 	getReq := &cns.GetInterfaceForContainerRequest{
-		NetworkContainerID: "ethWebApp",
+		NetworkContainerID: name,
 	}
 
 	json.NewEncoder(&body).Encode(getReq)

cns/restserver/const.go

@@ -5,33 +5,34 @@ package restserver
 
 // Container Network Service remote API Contract.
 const (
-	Success                         = 0
-	UnsupportedNetworkType          = 1
-	InvalidParameter                = 2
-	UnsupportedEnvironment          = 3
-	UnreachableHost                 = 4
-	ReservationNotFound             = 5
-	MalformedSubnet                 = 8
-	UnreachableDockerDaemon         = 9
-	UnspecifiedNetworkName          = 10
-	NotFound                        = 14
-	AddressUnavailable              = 15
-	NetworkContainerNotSpecified    = 16
-	CallToHostFailed                = 17
-	UnknownContainerID              = 18
-	UnsupportedOrchestratorType     = 19
-	DockerContainerNotSpecified     = 20
-	UnsupportedVerb                 = 21
-	UnsupportedNetworkContainerType = 22
-	InvalidRequest                  = 23
-	NetworkJoinFailed               = 24
-	NetworkContainerPublishFailed   = 25
-	NetworkContainerUnpublishFailed = 26
-	InvalidPrimaryIPConfig          = 27
-	PrimaryCANotSame                = 28
-	InconsistentIPConfigState       = 29
-	InvalidSecondaryIPConfig        = 30
-	UnexpectedError                 = 99
+	Success                                 = 0
+	UnsupportedNetworkType                  = 1
+	InvalidParameter                        = 2
+	UnsupportedEnvironment                  = 3
+	UnreachableHost                         = 4
+	ReservationNotFound                     = 5
+	MalformedSubnet                         = 8
+	UnreachableDockerDaemon                 = 9
+	UnspecifiedNetworkName                  = 10
+	NotFound                                = 14
+	AddressUnavailable                      = 15
+	NetworkContainerNotSpecified            = 16
+	CallToHostFailed                        = 17
+	UnknownContainerID                      = 18
+	UnsupportedOrchestratorType             = 19
+	DockerContainerNotSpecified             = 20
+	UnsupportedVerb                         = 21
+	UnsupportedNetworkContainerType         = 22
+	InvalidRequest                          = 23
+	NetworkJoinFailed                       = 24
+	NetworkContainerPublishFailed           = 25
+	NetworkContainerUnpublishFailed         = 26
+	InvalidPrimaryIPConfig                  = 27
+	PrimaryCANotSame                        = 28
+	InconsistentIPConfigState               = 29
+	InvalidSecondaryIPConfig                = 30
+	NetworkContainerPendingStatePropagation = 31
+	UnexpectedError                         = 99
 )
 
 const (
@@ -42,4 +43,5 @@ const (
 	detach          = "Detach"
 	// Rest service state identifier for named lock
 	stateJoinedNetworks = "JoinedNetworks"
+	dncApiVersion       = "?api-version=2018-03-01"
 )