[NPM Lite] Default Deny Policy

cni/network/invoker.go

@@ -28,7 +28,8 @@ type IPAMAddConfig struct {
 type IPAMAddResult struct {
 	interfaceInfo map[string]network.InterfaceInfo
 	// ncResponse and host subnet prefix were moved into interface info
-	ipv6Enabled bool
+	ipv6Enabled    bool
+	defaultDenyACL []cni.KVPair
 }
 
 func (ipamAddResult IPAMAddResult) PrettyString() string {

cni/network/invoker_cns.go

@@ -55,6 +55,7 @@ type IPResultInfo struct {
 	skipDefaultRoutes  bool
 	routes             []cns.Route
 	pnpID              string
+	defaultDenyACL     []cni.KVPair
 }
 
 func (i IPResultInfo) MarshalLogObject(encoder zapcore.ObjectEncoder) error {
@@ -159,8 +160,8 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
 			skipDefaultRoutes:  response.PodIPInfo[i].SkipDefaultRoutes,
 			routes:             response.PodIPInfo[i].Routes,
 			pnpID:              response.PodIPInfo[i].PnPID,
+			defaultDenyACL:     response.PodIPInfo[i].DefaultDenyACL,
 		}
-
 		logger.Info("Received info for pod",
 			zap.Any("ipInfo", info),
 			zap.Any("podInfo", podInfo))
@@ -444,6 +445,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add
 				Gw:  ncgw,
 			})
 		}
+		addResult.defaultDenyACL = append(addResult.defaultDenyACL, info.defaultDenyACL...)
 		// if we have multiple infra ip result infos, we effectively append routes and ip configs to that same interface info each time
 		// the host subnet prefix (in ipv4 or ipv6) will always refer to the same interface regardless of which ip result info we look at
 		addResult.interfaceInfo[key] = network.InterfaceInfo{

cni/network/invoker_cns_test.go

@@ -521,7 +521,30 @@ func TestCNSIPAMInvoker_Add(t *testing.T) {
 		hostSubnetPrefix *net.IPNet
 		options          map[string]interface{}
 	}
+	valueOut := []byte(`{
+		"Type": "ACL",
+		"Action": "Block",
+		"Direction": "Out",
+		"Priority": 10000
+	}`)
 
+	valueIn := []byte(`{
+		"Type": "ACL",
+		"Action": "Block",
+		"Direction": "In",
+		"Priority": 10000
+	}`)
+
+	expectedDefaultDenyACL := []cni.KVPair{
+		{
+			Name:  "EndpointPolicy",
+			Value: valueOut,
+		},
+		{
+			Name:  "EndpointPolicy",
+			Value: valueIn,
+		},
+	}
 	tests := []struct {
 		name                  string
 		fields                fields
@@ -559,7 +582,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) {
 										PrimaryIP: "10.0.0.1",
 										Subnet:    "10.0.0.0/24",
 									},
-									NICType: cns.InfraNIC,
+									NICType:        cns.InfraNIC,
+									DefaultDenyACL: expectedDefaultDenyACL,
 								},
 							},
 							Response: cns.Response{
@@ -628,6 +652,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) {
 										PrimaryIP: "10.0.0.1",
 										Subnet:    "10.0.0.0/24",
 									},
+									DefaultDenyACL: expectedDefaultDenyACL,
 								},
 							},
 							Response: cns.Response{
@@ -696,7 +721,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) {
 										PrimaryIP: "10.0.0.1",
 										Subnet:    "10.0.0.0/24",
 									},
-									NICType: cns.InfraNIC,
+									NICType:        cns.InfraNIC,
+									DefaultDenyACL: expectedDefaultDenyACL,
 								},
 								{
 									PodIPConfig: cns.IPSubnet{
@@ -795,8 +821,10 @@ func TestCNSIPAMInvoker_Add(t *testing.T) {
 			ipamAddResult, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options})
 			if tt.wantErr {
 				require.Error(err)
+				require.Equalf([]cni.KVPair(nil), ipamAddResult.defaultDenyACL, "incorrect default deny ACL")
 			} else {
 				require.NoError(err)
+				require.Equalf(expectedDefaultDenyACL, ipamAddResult.defaultDenyACL, "incorrect default deny ACL")
 			}
 
 			for _, ifInfo := range ipamAddResult.interfaceInfo {

cni/network/network.go

@@ -589,7 +589,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 		// sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo))
 	}
 
+	nwCfg.AdditionalArgs = append(nwCfg.AdditionalArgs, ipamAddResult.defaultDenyACL...)
 	policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs)
+
 	// moved to addIpamInvoker
 	// sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString()))
 

cns/NetworkContainerContract.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/google/uuid"
@@ -503,6 +504,8 @@ type PodIpInfo struct {
 	Routes []Route
 	// PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID
 	PnPID string
+	// Defauly Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes
+	DefaultDenyACL []cni.KVPair
 }
 
 type HostIPInfo struct {

cns/middlewares/k8sSwiftV2.go

@@ -40,7 +40,7 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)
 // and release IP configs handlers.
 func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
 	return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
-		podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)
+		podInfo, respCode, message, defaultDenyACLbool := k.validateIPConfigsRequest(ctx, &req)
 
 		if respCode != types.Success {
 			return &cns.IPConfigsResponse{
@@ -51,10 +51,24 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
 			}, errors.New("failed to validate IP configs request")
 		}
 		ipConfigsResp, err := defaultHandler(ctx, req)
+
 		// If the pod is not v2, return the response from the handler
 		if !req.SecondaryInterfacesExist {
 			return ipConfigsResp, err
 		}
+
+		// ipConfigsResp has infra IP configs -> if defaultDenyACLbool is enabled, add the default deny acl's pn the infra IP configs
+		for i := range ipConfigsResp.PodIPInfo {
+			ipInfo := &ipConfigsResp.PodIPInfo[i]
+			// there will be no pod connectivity to and from those pods
+			if defaultDenyACLbool {
+				err = addDefaultDenyACL(ipInfo)
+				if err != nil {
+					logger.Errorf("failed to add default deny acl's for pod %v with err %v", podInfo.Name(), err)
+				}
+			}
+		}
+
 		// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
 		defer func() {
 			// Release the default IP config if there is an error
@@ -102,36 +116,43 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
 
 // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
 // nolint
-func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
+func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string, defaultDenyACL bool) {
+	defaultDenyACLbool := false
+
 	// Retrieve the pod from the cluster
 	podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
 	if err != nil {
 		errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
-		return nil, types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool
 	}
 	logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
 	podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 	pod := v1.Pod{}
 	if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
 		errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
-		return nil, types.UnexpectedError, errBuf.Error()
+		return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool
 	}
 
 	// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
 	_, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2]
 	_, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2]
+
 	if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel {
 
 		// Check if the MTPNC CRD exists for the pod, if not, return error
 		mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
 		mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
 		if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
-			return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
+			return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error(), defaultDenyACLbool
 		}
 		// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
 		if !mtpnc.IsReady() {
-			return nil, types.UnexpectedError, errMTPNCNotReady.Error()
+			return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool
 		}
+
+		// setting defaultDenyACLbool from mtpnc
+		defaultDenyACLbool = mtpnc.Status.DefaultDenyACL
+
 		// If primary Ip is set in status field, it indicates the presence of secondary interfaces
 		if mtpnc.Status.PrimaryIP != "" {
 			req.SecondaryInterfacesExist = true
@@ -140,7 +161,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 		for _, interfaceInfo := range interfaceInfos {
 			if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
 				if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
-					return nil, types.UnexpectedError, errMTPNCNotReady.Error()
+					return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool
 				}
 				req.BackendInterfaceExist = true
 				req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)
@@ -154,7 +175,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 	logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
 	logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist)
 	// retrieve podinfo from orchestrator context
-	return podInfo, types.Success, ""
+	return podInfo, types.Success, "", defaultDenyACLbool
 }
 
 // getIPConfig returns the pod's SWIFT V2 IP configuration.

cns/middlewares/k8sSwiftV2_linux.go

@@ -103,3 +103,7 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo,
 }
 
 func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {}
+
+func addDefaultDenyACL(_ *cns.PodIpInfo) error {
+	return nil
+}

cns/middlewares/k8sSwiftV2_linux_test.go

@@ -144,7 +144,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq.OrchestratorContext = b
 	happyReq.SecondaryInterfacesExist = false
 
-	_, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq)
+	_, respCode, err, _ := middleware.validateIPConfigsRequest(context.TODO(), happyReq)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
@@ -158,7 +158,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq2.OrchestratorContext = b
 	happyReq2.SecondaryInterfacesExist = false
 
-	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
+	_, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
@@ -172,7 +172,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	happyReq3.OrchestratorContext = b
 	happyReq3.SecondaryInterfacesExist = false
 
-	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
+	_, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq3.SecondaryInterfacesExist, false)
@@ -188,7 +188,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
 		InfraContainerID: testPod1Info.InfraContainerID(),
 	}
 	failReq.OrchestratorContext = []byte("invalid")
-	_, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// Pod doesn't exist in cache test
@@ -198,19 +198,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
 	}
 	b, _ := testPod2Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// Failed to get MTPNC
 	b, _ = testPod3Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 
 	// MTPNC not ready
 	b, _ = testPod4Info.OrchestratorContext()
 	failReq.OrchestratorContext = b
-	_, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
+	_, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq)
 	assert.Equal(t, respCode, types.UnexpectedError)
 }
 

cns/middlewares/k8sSwiftV2_windows.go

@@ -1,9 +1,14 @@
 package middlewares
 
 import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/middlewares/utils"
 	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/pkg/errors"
 )
 
@@ -58,3 +63,51 @@
 	}
 	podIPInfo.Routes = append(podIPInfo.Routes, route)
 }
+
+func addDefaultDenyACL(podIPInfo *cns.PodIpInfo) error {
+	valueOut, err := getDefaultDenyACLPolicy(hcn.DirectionTypeOut)
+	if err != nil {
+		fmt.Printf("Failed to get default deny ACL policy egress: %v\n", err)
+		return err
+	}
+
+	valueIn, err := getDefaultDenyACLPolicy(hcn.DirectionTypeIn)
+	if err != nil {
+		fmt.Printf("Failed to get default deny ACL policy ingress: %v\n", err)
+		return err
+	}
+	additionalArgs := []cni.KVPair{
+		{
+			Name:  "EndpointPolicy",
+			Value: valueOut,
+		},
+		{
+			Name:  "EndpointPolicy",
+			Value: valueIn,
+		},
+	}
+	podIPInfo.DefaultDenyACL = append(podIPInfo.DefaultDenyACL, additionalArgs...)
+	return nil
+}
+
+func getDefaultDenyACLPolicy(direction hcn.DirectionType) ([]byte, error) {
+	const DefaultDenyPriority = 10000
+	type DefaultDenyACL struct {
+		Type      string            `json:"Type"`
+		Action    hcn.ActionType    `json:"Action"`
+		Direction hcn.DirectionType `json:"Direction"`
+		Priority  int               `json:"Priority"`
+	}
+	denyACL := DefaultDenyACL{
+		Type:      "ACL",
+		Action:    hcn.ActionTypeBlock,
+		Direction: direction,
+		Priority:  DefaultDenyPriority,
+	}
+	denyACLJSON, err := json.Marshal(denyACL)
+	if err != nil {
+		fmt.Println("Error marshaling default deny policy:", err)
+		return nil, err
+	}
+	return denyACLJSON, nil
+}