diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 9e766d020c..369ce46f77 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -28,7 +28,8 @@ type IPAMAddConfig struct { type IPAMAddResult struct { interfaceInfo map[string]network.InterfaceInfo // ncResponse and host subnet prefix were moved into interface info - ipv6Enabled bool + ipv6Enabled bool + defaultDenyACL []cni.KVPair } func (ipamAddResult IPAMAddResult) PrettyString() string { diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 02e0ce7351..1a81eeba83 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -55,6 +55,7 @@ type IPResultInfo struct { skipDefaultRoutes bool routes []cns.Route pnpID string + defaultDenyACL []cni.KVPair } func (i IPResultInfo) MarshalLogObject(encoder zapcore.ObjectEncoder) error { @@ -159,8 +160,8 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro skipDefaultRoutes: response.PodIPInfo[i].SkipDefaultRoutes, routes: response.PodIPInfo[i].Routes, pnpID: response.PodIPInfo[i].PnPID, + defaultDenyACL: response.PodIPInfo[i].DefaultDenyACL, } - logger.Info("Received info for pod", zap.Any("ipInfo", info), zap.Any("podInfo", podInfo)) @@ -444,6 +445,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add Gw: ncgw, }) } + addResult.defaultDenyACL = append(addResult.defaultDenyACL, info.defaultDenyACL...) // if we have multiple infra ip result infos, we effectively append routes and ip configs to that same interface info each time // the host subnet prefix (in ipv4 or ipv6) will always refer to the same interface regardless of which ip result info we look at addResult.interfaceInfo[key] = network.InterfaceInfo{ diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index c2d4963151..67b8a533d8 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -521,7 +521,30 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { hostSubnetPrefix *net.IPNet options map[string]interface{} } + valueOut := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "Out", + "Priority": 10000 + }`) + valueIn := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "In", + "Priority": 10000 + }`) + + expectedDefaultDenyACL := []cni.KVPair{ + { + Name: "EndpointPolicy", + Value: valueOut, + }, + { + Name: "EndpointPolicy", + Value: valueIn, + }, + } tests := []struct { name string fields fields @@ -559,7 +582,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + DefaultDenyACL: expectedDefaultDenyACL, }, }, Response: cns.Response{ @@ -628,6 +652,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, + DefaultDenyACL: expectedDefaultDenyACL, }, }, Response: cns.Response{ @@ -696,7 +721,8 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - NICType: cns.InfraNIC, + NICType: cns.InfraNIC, + DefaultDenyACL: expectedDefaultDenyACL, }, { PodIPConfig: cns.IPSubnet{ @@ -795,8 +821,10 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { ipamAddResult, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options}) if tt.wantErr { require.Error(err) + require.Equalf([]cni.KVPair(nil), ipamAddResult.defaultDenyACL, "incorrect default deny ACL") } else { require.NoError(err) + require.Equalf(expectedDefaultDenyACL, ipamAddResult.defaultDenyACL, "incorrect default deny ACL") } for _, ifInfo := range ipamAddResult.interfaceInfo { diff --git a/cni/network/network.go b/cni/network/network.go index 6b0635e1c7..703b389df4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -589,7 +589,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.interfaceInfo[ifIndex], ipamAddResult.interfaceInfo)) } + nwCfg.AdditionalArgs = append(nwCfg.AdditionalArgs, ipamAddResult.defaultDenyACL...) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + // moved to addIpamInvoker // sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam interface: %+v", ipamAddResult.PrettyString())) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 394f871f09..18829c3f8b 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/google/uuid" @@ -503,6 +504,8 @@ type PodIpInfo struct { Routes []Route // PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID PnPID string + // Defauly Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes + DefaultDenyACL []cni.KVPair } type HostIPInfo struct { diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a11290c205..aa49d6a258 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -40,7 +40,7 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) // and release IP configs handlers. func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req) + podInfo, respCode, message, defaultDenyACLbool := k.validateIPConfigsRequest(ctx, &req) if respCode != types.Success { return &cns.IPConfigsResponse{ @@ -51,10 +51,24 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa }, errors.New("failed to validate IP configs request") } ipConfigsResp, err := defaultHandler(ctx, req) + // If the pod is not v2, return the response from the handler if !req.SecondaryInterfacesExist { return ipConfigsResp, err } + + // ipConfigsResp has infra IP configs -> if defaultDenyACLbool is enabled, add the default deny acl's pn the infra IP configs + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // there will be no pod connectivity to and from those pods + if defaultDenyACLbool { + err = addDefaultDenyACL(ipInfo) + if err != nil { + logger.Errorf("failed to add default deny acl's for pod %v with err %v", podInfo.Name(), err) + } + } + } + // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config defer func() { // Release the default IP config if there is an error @@ -102,36 +116,43 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. // nolint -func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { +func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string, defaultDenyACL bool) { + defaultDenyACLbool := false + // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) - return nil, types.UnexpectedError, errBuf.Error() + return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool } logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) - return nil, types.UnexpectedError, errBuf.Error() + return nil, types.UnexpectedError, errBuf.Error(), defaultDenyACLbool } // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] + if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error(), defaultDenyACLbool } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if !mtpnc.IsReady() { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() + return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool } + + // setting defaultDenyACLbool from mtpnc + defaultDenyACLbool = mtpnc.Status.DefaultDenyACL + // If primary Ip is set in status field, it indicates the presence of secondary interfaces if mtpnc.Status.PrimaryIP != "" { req.SecondaryInterfacesExist = true @@ -140,7 +161,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req for _, interfaceInfo := range interfaceInfos { if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() + return nil, types.UnexpectedError, errMTPNCNotReady.Error(), defaultDenyACLbool } req.BackendInterfaceExist = true req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) @@ -154,7 +175,7 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist) // retrieve podinfo from orchestrator context - return podInfo, types.Success, "" + return podInfo, types.Success, "", defaultDenyACLbool } // getIPConfig returns the pod's SWIFT V2 IP configuration. diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index e9a93de0e2..99b8ae7846 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -103,3 +103,7 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, } func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + +func addDefaultDenyACL(_ *cns.PodIpInfo) error { + return nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux_test.go b/cns/middlewares/k8sSwiftV2_linux_test.go index 76be6b2149..8d42f8cfe1 100644 --- a/cns/middlewares/k8sSwiftV2_linux_test.go +++ b/cns/middlewares/k8sSwiftV2_linux_test.go @@ -144,7 +144,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.SecondaryInterfacesExist = false - _, respCode, err := middleware.validateIPConfigsRequest(context.TODO(), happyReq) + _, respCode, err, _ := middleware.validateIPConfigsRequest(context.TODO(), happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -158,7 +158,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq2.OrchestratorContext = b happyReq2.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2) + _, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq2) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -172,7 +172,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq3.OrchestratorContext = b happyReq3.SecondaryInterfacesExist = false - _, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3) + _, respCode, err, _ = middleware.validateIPConfigsRequest(context.TODO(), happyReq3) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq3.SecondaryInterfacesExist, false) @@ -188,7 +188,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - _, respCode, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ := middleware.validateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -198,19 +198,19 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Failed to get MTPNC b, _ = testPod3Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // MTPNC not ready b, _ = testPod4Info.OrchestratorContext() failReq.OrchestratorContext = b - _, respCode, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) + _, respCode, _, _ = middleware.validateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) } diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index 2be2fbd1df..b679e9c286 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -1,9 +1,14 @@ package middlewares import ( + "encoding/json" + "fmt" + + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Microsoft/hcsshim/hcn" "github.com/pkg/errors" ) @@ -58,3 +63,51 @@ func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP st } podIPInfo.Routes = append(podIPInfo.Routes, route) } + +func addDefaultDenyACL(podIPInfo *cns.PodIpInfo) error { + valueOut, err := getDefaultDenyACLPolicy(hcn.DirectionTypeOut) + if err != nil { + fmt.Printf("Failed to get default deny ACL policy egress: %v\n", err) + return err + } + + valueIn, err := getDefaultDenyACLPolicy(hcn.DirectionTypeIn) + if err != nil { + fmt.Printf("Failed to get default deny ACL policy ingress: %v\n", err) + return err + } + additionalArgs := []cni.KVPair{ + { + Name: "EndpointPolicy", + Value: valueOut, + }, + { + Name: "EndpointPolicy", + Value: valueIn, + }, + } + podIPInfo.DefaultDenyACL = append(podIPInfo.DefaultDenyACL, additionalArgs...) + return nil +} + +func getDefaultDenyACLPolicy(direction hcn.DirectionType) ([]byte, error) { + const DefaultDenyPriority = 10000 + type DefaultDenyACL struct { + Type string `json:"Type"` + Action hcn.ActionType `json:"Action"` + Direction hcn.DirectionType `json:"Direction"` + Priority int `json:"Priority"` + } + denyACL := DefaultDenyACL{ + Type: "ACL", + Action: hcn.ActionTypeBlock, + Direction: direction, + Priority: DefaultDenyPriority, + } + denyACLJSON, err := json.Marshal(denyACL) + if err != nil { + fmt.Println("Error marshaling default deny policy:", err) + return nil, err + } + return denyACLJSON, nil +} diff --git a/cns/middlewares/k8sSwiftV2_windows_test.go b/cns/middlewares/k8sSwiftV2_windows_test.go index dab24685f9..2eb9ccd61f 100644 --- a/cns/middlewares/k8sSwiftV2_windows_test.go +++ b/cns/middlewares/k8sSwiftV2_windows_test.go @@ -1,12 +1,15 @@ package middlewares import ( + "encoding/json" "reflect" "testing" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/stretchr/testify/require" "gotest.tools/v3/assert" ) @@ -100,3 +103,73 @@ func TestAddDefaultRoute(t *testing.T) { t.Errorf("got '%+v', expected '%+v'", ipInfo.Routes, expectedRoutes) } } + +func TestAddDefaultDenyACL(t *testing.T) { + valueOut := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "Out", + "Priority": 10000 + }`) + + valueIn := []byte(`{ + "Type": "ACL", + "Action": "Block", + "Direction": "In", + "Priority": 10000 + }`) + + expectedDefaultDenyACL := []cni.KVPair{ + { + Name: "EndpointPolicy", + Value: valueOut, + }, + { + Name: "EndpointPolicy", + Value: valueIn, + }, + } + + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + } + + err := addDefaultDenyACL(&podIPInfo) + assert.Equal(t, err, nil) + + // Normalize both slices so there is no extra spacing, new lines, etc + normalizedExpected := normalizeKVPairs(t, expectedDefaultDenyACL) + normalizedActual := normalizeKVPairs(t, podIPInfo.DefaultDenyACL) + if !reflect.DeepEqual(normalizedExpected, normalizedActual) { + t.Errorf("got '%+v', expected '%+v'", podIPInfo.DefaultDenyACL, expectedDefaultDenyACL) + } +} + +// normalizeKVPairs normalizes the JSON values in the KV pairs by unmarshaling them into a map, then marshaling them back to compact JSON to remove any extra space, new lines, etc +func normalizeKVPairs(t *testing.T, kvPairs []cni.KVPair) []cni.KVPair { + normalized := make([]cni.KVPair, len(kvPairs)) + + for i, kv := range kvPairs { + var unmarshaledValue map[string]interface{} + // Unmarshal the Value into a map + err := json.Unmarshal(kv.Value, &unmarshaledValue) + require.NoError(t, err, "Failed to unmarshal JSON value") + + // Marshal it back to compact JSON + normalizedValue, err := json.Marshal(unmarshaledValue) + require.NoError(t, err, "Failed to re-marshal JSON value") + + // Replace Value with the normalized compact JSON + normalized[i] = cni.KVPair{ + Name: kv.Name, + Value: normalizedValue, + } + } + + return normalized +} diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index ab481fa496..3798027268 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -85,6 +85,8 @@ type MultitenantPodNetworkConfigStatus struct { // InterfaceInfos describes all of the network container goal state for this Pod // +kubebuilder:validation:Optional InterfaceInfos []InterfaceInfo `json:"interfaceInfos,omitempty"` + // DefaultDenyAcl bool indicates whether default deny policy will be present on the pods upon pod creation + DefaultDenyACL bool `json:"defaultDenyACL"` } func init() { diff --git a/crd/multitenancy/api/v1alpha1/podnetworkinstance.go b/crd/multitenancy/api/v1alpha1/podnetworkinstance.go index 4a775363ae..5b9d5c6751 100644 --- a/crd/multitenancy/api/v1alpha1/podnetworkinstance.go +++ b/crd/multitenancy/api/v1alpha1/podnetworkinstance.go @@ -56,6 +56,9 @@ type PodNetworkInstanceSpec struct { // optional for now in case orchestrator uses the deprecated fields // +kubebuilder:validation:Optional PodNetworkConfigs []PodNetworkConfig `json:"podNetworkConfigs"` + // DefaultDenyAcl bool indicates whether default deny policy will be present on the pods upon pod creation + // +kubebuilder:default=false + DefaultDenyACL bool `json:"defaultDenyACL"` } // PodNetworkInstanceStatus defines the observed state of PodNetworkInstance