updated CNS for adding default deny acl's

Author: rejain789

cns/NetworkContainerContract.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/google/uuid"
@@ -503,6 +504,8 @@ type PodIpInfo struct {
 	Routes []Route
 	// PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID
 	PnPID string
+	// Defauly Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes
+	DefaultDenyACL []cni.KVPair
 }
 
 type HostIPInfo struct {

cns/middlewares/k8sSwiftV2.go

@@ -40,7 +40,9 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)
 // and release IP configs handlers.
 func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
 	return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
-		podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)
+		podInfo, respCode, message, defaultDenyACLbool := k.validateIPConfigsRequest(ctx, &req)
+
+		logger.Printf("defaultDenyACLbool value is: %v", defaultDenyACLbool)
 
 		if respCode != types.Success {
 			return &cns.IPConfigsResponse{
@@ -55,6 +57,19 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
 		if !req.SecondaryInterfacesExist {
 			return ipConfigsResp, err
 		}
+
+		// ipConfigsResp has infra IP configs -> if defaultDenyACLbool is enabled, add the default deny acl's pn the infra IP configs
+		for i := range ipConfigsResp.PodIPInfo {
+			ipInfo := &ipConfigsResp.PodIPInfo[i]
+			// there will be no pod connectivity to and from those pods
+			if defaultDenyACLbool {
+				err = addDefaultDenyACL(ipInfo)
+				if err != nil {
+					logger.Errorf("failed to add default deny acl's for pod %v with err %v", podInfo.Name(), err)
+				}
+			}
+		}
+
 		// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
 		defer func() {
 			// Release the default IP config if there is an error
@@ -102,19 +117,21 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
 
 // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
 // nolint
-func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
+func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string, defaultDenyACL bool) {
+	defaultDenyACLbool := false
+
 	// Retrieve the pod from the cluster
 	podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
 	if err != nil {
 		errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
-		return nil, types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool
 	}
 	logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
 	podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 	pod := v1.Pod{}
 	if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
 		errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
-		return nil, types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool
 	}
 
 	// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
@@ -126,12 +143,16 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 		mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
 		mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 		if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
-			return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
+			return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error(), defaultDenyACLbool
 		}
 		// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
 		if !mtpnc.IsReady() {
-			return nil, types.UnexpectedError, errMTPNCNotReady.Error()
+			return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool
 		}
+
+		// copying defaultDenyACL bool from mtpnc
+		defaultDenyACLbool = mtpnc.Status.DefaultDenyACL
+
 		// If primary Ip is set in status field, it indicates the presence of secondary interfaces
 		if mtpnc.Status.PrimaryIP != "" {
 			req.SecondaryInterfacesExist = true
@@ -140,7 +161,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 		for _, interfaceInfo := range interfaceInfos {
 			if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
 				if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
-					return nil, types.UnexpectedError, errMTPNCNotReady.Error()
+					return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool
 				}
 				req.BackendInterfaceExist = true
 				req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)
@@ -154,7 +175,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 	logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
 	logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist)
 	// retrieve podinfo from orchestrator context
-	return podInfo, types.Success, ""
+	return podInfo, types.Success, "", defaultDenyACLbool
 }
 
 // getIPConfig returns the pod's SWIFT V2 IP configuration.

cns/middlewares/k8sSwiftV2_linux.go

@@ -103,3 +103,7 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo,
 }
 
 func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {}
+
+func addDefaultDenyACL(_ *cns.PodIpInfo) error {
+	return nil
+}

cns/middlewares/k8sSwiftV2_linux_test.go

@@ -144,7 +144,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq.OrchestratorContext = b
 	happyReq.SecondaryInterfacesExist = false
 
-	_, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq)
+	_, respCode, err, _ := middleware.validateIPConfigsRequest(context.TODO(), happyReq)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
@@ -158,7 +158,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq2.OrchestratorContext = b
 	happyReq2.SecondaryInterfacesExist = false
 
-	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
+	_, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
@@ -172,7 +172,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq3.OrchestratorContext = b
 	happyReq3.SecondaryInterfacesExist = false
 
-	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
+	_, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq3.SecondaryInterfacesExist, false)
@@ -188,7 +188,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
 		InfraContainerID: testPod1Info.InfraContainerID(),
 	}
 	failReq.OrchestratorContext = []byte("invalid")
-	_, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// Pod doesn't exist in cache test
@@ -198,19 +198,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
 	}
 	b, _ := testPod2Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// Failed to get MTPNC
 	b, _ = testPod3Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// MTPNC not ready
 	b, _ = testPod4Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 }
 

cns/middlewares/k8sSwiftV2_windows.go

@@ -1,9 +1,14 @@
 package middlewares
 
 import (
+	"encoding/json"
+
+	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/middlewares/utils"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/pkg/errors"
 )
 
@@ -58,3 +63,62 @@ func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP st
 	}
 	podIPInfo.Routes = append(podIPInfo.Routes, route)
 }
+
+// append the default deny acl's to the list defaultDenyACL field in podIpInfo
+func addDefaultDenyACL(podIpInfo *cns.PodIpInfo) error {
+	blockEgressACL, err := getDefaultDenyACLPolicy(hcn.DirectionTypeOut)
+	if err != nil {
+		return errors.Wrap(err, "Failed to create default deny ACL policy egress")
+	}
+
+	blockIngressACL, err := getDefaultDenyACLPolicy(hcn.DirectionTypeIn)
+	if err != nil {
+		return errors.Wrap(err, "Failed to create default deny ACL policy ingress")
+	}
+
+	additionalArgs := []cni.KVPair{
+		{
+			Name:  "EndpointPolicy",
+			Value: blockEgressACL,
+		},
+		{
+			Name:  "EndpointPolicy",
+			Value: blockIngressACL,
+		},
+	}
+
+	podIpInfo.DefaultDenyACL = append(podIpInfo.DefaultDenyACL, additionalArgs...)
+
+	logger.Printf("The length of podIpInfo.DefaultDenyACL is: %v", len(podIpInfo.DefaultDenyACL))
+
+	return nil
+}
+
+// create the default deny acl's that need to be added to the list defaultDenyACL field in podIpInfo
+func getDefaultDenyACLPolicy(direction hcn.DirectionType) ([]byte, error) {
+	const DefaultDenyPriority = 10000
+	const policyType = "ACL"
+	type DefaultDenyACL struct {
+		Type      string            `json:"Type"`
+		Action    hcn.ActionType    `json:"Action"`
+		Direction hcn.DirectionType `json:"Direction"`
+		Priority  int               `json:"Priority"`
+	}
+
+	denyACL := DefaultDenyACL{
+		Type:      policyType,
+		Action:    hcn.ActionTypeBlock,
+		Direction: direction,
+		Priority:  DefaultDenyPriority,
+	}
+
+	denyACLJSON, err := json.Marshal(denyACL)
+
+	logger.Printf("ACL Created for direction %s is : %s", direction, denyACLJSON)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "error marshalling default deny policy to json")
+	}
+
+	return denyACLJSON, nil
+}

crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go

@@ -85,6 +85,8 @@ type MultitenantPodNetworkConfigStatus struct {
 	// InterfaceInfos describes all of the network container goal state for this Pod
 	// +kubebuilder:validation:Optional
 	InterfaceInfos []InterfaceInfo `json:"interfaceInfos,omitempty"`
+	// DefaultDenyAcl bool indicates whether default deny policy will be present on the pods upon pod creation
+	DefaultDenyACL bool `json:"defaultDenyACL"`
 }
 
 func init() {

crd/multitenancy/api/v1alpha1/podnetworkinstance.go

@@ -56,6 +56,9 @@ type PodNetworkInstanceSpec struct {
 	// optional for now in case orchestrator uses the deprecated fields
 	// +kubebuilder:validation:Optional
 	PodNetworkConfigs []PodNetworkConfig `json:"podNetworkConfigs"`
+	// DefaultDenyAcl bool indicates whether default deny policy will be present on the pods upon pod creation
+	// +kubebuilder:default=false
+	DefaultDenyACL bool `json:"defaultDenyACL"`
 }
 
 // PodNetworkInstanceStatus defines the observed state of PodNetworkInstance