refactor: SWIFT v2 Middlewares (#2390)

* refractor: swift v2 middleware

* revert: unexport getTestService

* fix: linter

* fix: linter issue

* refactor: generic renaming of middlewares-related code in cns

* refactor: change type of swiftv2middleware

* address feedbacks

* rename for clarity

* missed some renaming

* renaming

* addressed comments

* add UT for no-op releasing ip config for v2 pod even if its mtpnc is not ready

* pass in SWIFTv2Mode as string

* typed SWIFTV2Mode

Author: nddq

cns/api.go

@@ -49,17 +49,16 @@ type HTTPService interface {
 	GetPendingReleaseIPConfigs() []IPConfigurationStatus
 	GetPodIPConfigState() map[string]IPConfigurationStatus
 	MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error)
-	AttachSWIFTv2Middleware(middleware SWIFTv2Middleware)
+	AttachIPConfigsHandlerMiddleware(IPConfigsHandlerMiddleware)
 }
 
-// Middleware interface for testing later on
-type SWIFTv2Middleware interface {
-	ValidateIPConfigsRequest(context.Context, *IPConfigsRequest) (types.ResponseCode, string)
-	GetIPConfig(context.Context, PodInfo) (PodIpInfo, error)
-	SetRoutes(*PodIpInfo) error
-}
+// IPConfigsHandlerFunc
+type IPConfigsHandlerFunc func(context.Context, IPConfigsRequest) (*IPConfigsResponse, error)
 
-type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string)
+// IPConfigsHandlerMiddleware
+type IPConfigsHandlerMiddleware interface {
+	IPConfigsRequestHandlerWrapper(defaultHandler IPConfigsHandlerFunc, failureHandler IPConfigsHandlerFunc) IPConfigsHandlerFunc
+}
 
 // This is used for KubernetesCRD orchestrator Type where NC has multiple ips.
 // This struct captures the state for SecondaryIPs associated to a given NC

cns/configuration/configuration.go

@@ -13,17 +13,24 @@ import (
 	"github.com/pkg/errors"
 )
 
+type SWIFTV2Mode string
+
 const (
 	// EnvCNSConfig is the CNS_CONFIGURATION_PATH env var key
 	EnvCNSConfig      = "CNS_CONFIGURATION_PATH"
 	defaultConfigName = "cns_config.json"
+	// Service Fabric SWIFTV2 mode
+	SFSWIFTV2 SWIFTV2Mode = "SFSWIFTV2"
+	// K8s SWIFTV2 mode
+	K8sSWIFTV2 SWIFTV2Mode = "K8sSWIFTV2"
 )
 
 type CNSConfig struct {
 	ChannelMode                 string
 	EnablePprof                 bool
 	EnableSubnetScarcity        bool
 	EnableSwiftV2               bool
+	SWIFTV2Mode                 SWIFTV2Mode
 	InitializeFromCNI           bool
 	ManagedSettings             ManagedSettings
 	MetricsBindAddress          string

cns/fakes/cnsfake.go

@@ -277,4 +277,4 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error {
 
 func (fake *HTTPServiceFake) Stop() {}
 
-func (fake *HTTPServiceFake) AttachSWIFTv2Middleware(cns.SWIFTv2Middleware) {}
+func (fake *HTTPServiceFake) AttachIPConfigsHandlerMiddleware(cns.IPConfigsHandlerMiddleware) {}

cns/middlewares/swiftV2.go renamed to cns/middlewares/k8sSwiftV2.go

@@ -30,25 +30,89 @@ const (
 	overlayGatewayV6 = "fe80::1234:5678:9abc"
 )
 
-type SWIFTv2Middleware struct {
+type K8sSWIFTv2Middleware struct {
 	Cli client.Client
 }
 
-// ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario.
+// Verify interface compliance at compile time
+var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)
+
+// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
+// and release IP configs handlers.
+func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
+	return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
+		podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req)
+
+		if respCode != types.Success {
+			return &cns.IPConfigsResponse{
+				Response: cns.Response{
+					ReturnCode: respCode,
+					Message:    message,
+				},
+			}, errors.New("failed to validate ip configs request")
+		}
+		ipConfigsResp, err := defaultHandler(ctx, req)
+		// If the pod is not v2, return the response from the handler
+		if !req.SecondaryInterfacesExist {
+			return ipConfigsResp, err
+		}
+		// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
+		defer func() {
+			// Release the default IP config if there is an error
+			if err != nil {
+				_, err = failureHandler(ctx, req)
+				if err != nil {
+					logger.Errorf("failed to release default IP config : %v", err)
+				}
+			}
+		}()
+		if err != nil {
+			return ipConfigsResp, err
+		}
+		SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo)
+		if err != nil {
+			return &cns.IPConfigsResponse{
+				Response: cns.Response{
+					ReturnCode: types.FailedToAllocateIPConfig,
+					Message:    fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
+				},
+				PodIPInfo: []cns.PodIpInfo{},
+			}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
+		}
+		ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo)
+		// Set routes for the pod
+		for i := range ipConfigsResp.PodIPInfo {
+			ipInfo := &ipConfigsResp.PodIPInfo[i]
+			err = m.setRoutes(ipInfo)
+			if err != nil {
+				return &cns.IPConfigsResponse{
+					Response: cns.Response{
+						ReturnCode: types.FailedToAllocateIPConfig,
+						Message:    fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
+					},
+					PodIPInfo: []cns.PodIpInfo{},
+				}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
+			}
+		}
+		return ipConfigsResp, nil
+	}
+}
+
+// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
 // nolint
-func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) {
+func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
 	// Retrieve the pod from the cluster
 	podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
 	if err != nil {
 		errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
-		return types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error()
 	}
 	logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
 	podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 	pod := v1.Pod{}
 	if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
 		errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
-		return types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error()
 	}
 
 	// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
@@ -58,19 +122,20 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c
 		mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
 		mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 		if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
-			return types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
+			return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
 		}
 		// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
 		if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
-			return types.UnexpectedError, errMTPNCNotReady.Error()
+			return nil, types.UnexpectedError, errMTPNCNotReady.Error()
 		}
 	}
 	logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
-	return types.Success, ""
+	// retrieve podinfo from orchestrator context
+	return podInfo, types.Success, ""
 }
 
-// GetIPConfig returns the pod's SWIFT V2 IP configuration.
-func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
+// getIPConfig returns the pod's SWIFT V2 IP configuration.
+func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
 	// Check if the MTPNC CRD exists for the pod, if not, return error
 	mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
 	mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
@@ -109,8 +174,8 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo
 	return podIPInfo, nil
 }
 
-// SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
-func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
+// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
+func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 	logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
 	podIPInfo.Routes = []cns.Route{}
 	switch podIPInfo.NICType {