backport "feat: Consume the NCStatus to be able to append subnet is full error to Pod IP reservation failures" to release/v1.4 (#2252)

* Feat: Created a Network Container Status Section to be updated with the latest error code (#2193)

* Added Network Conatiner Status to include the latest error code for a Network Container

* Updated the crd to have the Status field included into the Network Container

* Updated the names and added Status and ErrorText as two fields in NC Status

* Fixed the casing and json values for these variables

* Added error code to the NC Status and removed the latest prefix from the varibale names

* Removed the timestamp variable from the NC Status

* Moved the Status object inside the NC Status to be able to accurately define the status of each NC for the node

* Changed to having an enum representing the NC Status which DNC-RC will update after inferring the error and CNS can use this field to propagate and NCRequest failures

* Made the validation of the new enum optional to keep it backward compatible

* feat: Consume the NCStatus to be able to append subnet is full error to Pod IP reservation failures (#2202)

* Added Network Conatiner Status to include the latest error code for a Network Container

* Updated the crd to have the Status field included into the Network Container

* Updated the names and added Status and ErrorText as two fields in NC Status

* Fixed the casing and json values for these variables

* Propagated the NC Status inside the CNS and IPAM Monitor pool states

* Fixed the lint error of missing comma

* Saved the updated NC Status into the CNS statefile

* Updated the IP assignment to check and error out subnet is Full when there are no more available IPs for CNS to assign

* Fixed a minor compilation issue

* Fixed lint failures

* Fixed lint failures

* Removed the reference from the metastate of the ipam monitor

* Added Update Success and Update Failed statuses to the NC Status to be able to clearly indicate response status inside the NNC from DNC-RC

* Updated the error to use errors pkg instead of fmt

* Updating the cns reconcillation logic to skip if there is a failure updating the NC and there are no IPs allocated for the NC

* Handled PR comments:
* Updated the code to have the NC status be part of the error directly so that it can be consumed by containerD
 and cx can perform actions on it.
 * Code update to not use dynamic slices.
 * Removed the logic which handled 0 IPs allocated to NNC in CNS reconcile

Signed-off-by: GitHub <[email protected]>

* Addressed the PR comment which helped delete a block of code to store ncIDs and also added more error codes to the NCStatus

---------

Signed-off-by: GitHub <[email protected]>

* Fixed all the compilation failures in the files after chery-pick

---------

Signed-off-by: GitHub <[email protected]>

Author: nairashu

cns/NetworkContainerContract.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -92,16 +93,17 @@ type CreateNetworkContainerRequest struct {
 	AllowHostToNCCommunication bool
 	AllowNCToHostCommunication bool
 	EndpointPolicies           []NetworkContainerRequestPolicies
+	NCStatus                   v1alpha.NCStatus
 }
 
 // CreateNetworkContainerRequest implements fmt.Stringer for logging
 func (req *CreateNetworkContainerRequest) String() string {
 	return fmt.Sprintf("CreateNetworkContainerRequest"+
 		"{Version: %s, NetworkContainerType: %s, NetworkContainerid: %s, PrimaryInterfaceIdentifier: %s, "+
 		"LocalIPConfiguration: %+v, IPConfiguration: %+v, SecondaryIPConfigs: %+v, MultitenancyInfo: %+v, "+
-		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t}",
+		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t, NCStatus: %s}",
 		req.Version, req.NetworkContainerType, req.NetworkContainerid, req.PrimaryInterfaceIdentifier, req.LocalIPConfiguration,
-		req.IPConfiguration, req.SecondaryIPConfigs, req.MultiTenancyInfo, req.AllowHostToNCCommunication, req.AllowNCToHostCommunication)
+		req.IPConfiguration, req.SecondaryIPConfigs, req.MultiTenancyInfo, req.AllowHostToNCCommunication, req.AllowNCToHostCommunication, string(req.NCStatus))
 }
 
 // NetworkContainerRequestPolicies - specifies policies associated with create network request

cns/kubecontroller/nodenetworkconfig/conversion.go

@@ -66,6 +66,7 @@ func CreateNCRequestFromDynamicNC(nc v1alpha.NetworkContainer) (*cns.CreateNetwo
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}, nil
 }
 

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -32,5 +32,6 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}
 }

cns/kubecontroller/nodenetworkconfig/conversion_windows.go

@@ -44,5 +44,6 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}
 }

cns/restserver/ipam.go

@@ -542,7 +542,9 @@ func (service *HTTPRestService) AssignAnyAvailableIPConfig(podInfo cns.PodInfo)
 	service.Lock()
 	defer service.Unlock()
 
+	var ncID string
 	for _, ipState := range service.PodIPConfigState {
+		ncID = ipState.NCID
 		if ipState.GetState() == types.Available {
 			if err := service.assignIPConfig(ipState, podInfo); err != nil {
 				return cns.PodIpInfo{}, err
@@ -557,7 +559,8 @@ func (service *HTTPRestService) AssignAnyAvailableIPConfig(podInfo cns.PodInfo)
 		}
 	}
 	//nolint:goerr113
-	return cns.PodIpInfo{}, fmt.Errorf("no IPs available, waiting on Azure CNS to allocate more")
+	return cns.PodIpInfo{}, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s",
+		ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 }
 
 // If IPConfig is already assigned to pod, it returns that else it returns one of the available ipconfigs.

crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go

@@ -58,6 +58,19 @@ const (
 	Error    Status = "Error"
 )
 
+// NCStatus indicates the latest NC request status
+// +kubebuilder:validation:Enum=NCUpdateSubnetFullError;NCUpdateInternalServerError;NCUpdateUnauthorizedError;NCUpdateSuccess;NCUpdateFailed
+// +kubebuilder:validation:Optional
+type NCStatus string
+
+const (
+	NCUpdateSubnetFull          NCStatus = "NCUpdateSubnetFullError"
+	NCUpdateInternalServerError NCStatus = "NCUpdateInternalServerError"
+	NCUpdateUnauthorizedError   NCStatus = "NCUpdateUnauthorizedError"
+	NCUpdateSuccess             NCStatus = "NCUpdateSuccess"
+	NCUpdateFailed              NCStatus = "NCUpdateFailed"
+)
+
 // NodeNetworkConfigStatus defines the observed state of NetworkConfig
 type NodeNetworkConfigStatus struct {
 	// +kubebuilder:default=0
@@ -107,12 +120,13 @@ type NetworkContainer struct {
 	SubnetAddressSpace string         `json:"subnetAddressSpace,omitempty"`
 	// +kubebuilder:default=0
 	// +kubebuilder:validation:Optional
-	Version         int64  `json:"version"`
-	NodeIP          string `json:"nodeIP,omitempty"`
-	SubscriptionID  string `json:"subcriptionID,omitempty"`
-	ResourceGroupID string `json:"resourceGroupID,omitempty"`
-	VNETID          string `json:"vnetID,omitempty"`
-	SubnetID        string `json:"subnetID,omitempty"`
+	Version         int64    `json:"version"`
+	NodeIP          string   `json:"nodeIP,omitempty"`
+	SubscriptionID  string   `json:"subcriptionID,omitempty"`
+	ResourceGroupID string   `json:"resourceGroupID,omitempty"`
+	VNETID          string   `json:"vnetID,omitempty"`
+	SubnetID        string   `json:"subnetID,omitempty"`
+	Status          NCStatus `json:"status,omitempty"`
 }
 
 // IPAssignment groups an IP address and Name. Name is a UUID set by the the IP address assigner.

crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml

@@ -115,6 +115,15 @@ spec:
                       type: string
                     resourceGroupID:
                       type: string
+                    status:
+                      description: NCStatus indicates the latest NC request status
+                      enum:
+                      - NCUpdateSubnetFullError
+                      - NCUpdateInternalServerError
+                      - NCUpdateUnauthorizedError
+                      - NCUpdateSuccess
+                      - NCUpdateFailed
+                      type: string
                     subcriptionID:
                       type: string
                     subnetAddressSpace: