Skip to content

Commit 313ce0c

Browse files
shreyashastantramshreyashastantram
authored
feat: Adding backend interfaces support to CNS (#2693)
* Adding backend interfaces support for CNS * Fixing the log line * Moving backend nic functions to platform package * Moving the pnpID fetching to platform package * Moving the mapping to CRD mode * Moving macaddress parsing to net package function * removing the backend nic config * Handling multiple interface information * Adding error handling for macaddress parsing * Moving pnpid mapping to swift v2 flow * Adding pipeline vars * Adding ut's for Happy paths * Adding ut's for Happy paths * Adding mtpnc check * Rebasing with master * Addressing lint errors * Adding function to linux file * Addressing lint errors * Addressing lint comments * Addressing Lint issues * Modifying ut's * removing mutex lock on cache * Adding podnetwork instance label * Modifying ut's after change * Moving function out of restserver * Moving mapping after crd intial state * Adding the config debug statements * removing the unused config * Adding the debug statements * Adding the debug statements * Adding more debug logs * removing log lines and adding mtpnc status check * Moving mapping to state file * Rebasing and removing the debug statements * Addressing lint errors * removing the debug pipeline changes * Adding more coverage * Adding test coverage and unhappy paths * Addressing lint errors * addressing lint errors * Addressing lint errors
1 parent 438036f commit 313ce0c

File tree

18 files changed

+589
-92
lines changed

18 files changed

+589
-92
lines changed

‎cns/NetworkContainerContract.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,8 @@ type PodIpInfo struct {
481481
SkipDefaultRoutes bool
482482
// Routes to configure on interface
483483
Routes []Route
484+
// PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID
485+
PnPID string
484486
}
485487

486488
type HostIPInfo struct {
@@ -499,12 +501,14 @@ type IPConfigRequest struct {
499501

500502
// Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice
501503
type IPConfigsRequest struct {
502-
DesiredIPAddresses []string `json:"desiredIPAddresses"`
503-
PodInterfaceID string `json:"podInterfaceID"`
504-
InfraContainerID string `json:"infraContainerID"`
505-
OrchestratorContext json.RawMessage `json:"orchestratorContext"`
506-
Ifname string `json:"ifname"` // Used by delegated IPAM
507-
SecondaryInterfacesExist bool `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func
504+
DesiredIPAddresses []string `json:"desiredIPAddresses"`
505+
PodInterfaceID string `json:"podInterfaceID"`
506+
InfraContainerID string `json:"infraContainerID"`
507+
OrchestratorContext json.RawMessage `json:"orchestratorContext"`
508+
Ifname string `json:"ifname"` // Used by delegated IPAM
509+
SecondaryInterfacesExist bool `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func
510+
BackendInterfaceExist bool `json:"BackendInterfaceExist"` // will be set by SWIFT v2 validator func
511+
BackendInterfaceMacAddresses []string `json:"BacknendInterfaceMacAddress"`
508512
}
509513

510514
// IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD

‎cns/configuration/env.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@ const (
1414
// LabelNodeSwiftV2 is the Node label for Swift V2
1515
LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled"
1616
// LabelPodSwiftV2 is the Pod label for Swift V2
17-
LabelPodSwiftV2 = "kubernetes.azure.com/pod-network"
18-
EnvPodCIDRs = "POD_CIDRs"
19-
EnvServiceCIDRs = "SERVICE_CIDRs"
20-
EnvInfraVNETCIDRs = "INFRA_VNET_CIDRs"
17+
LabelPodSwiftV2 = "kubernetes.azure.com/pod-network"
18+
LabelPodNetworkInstanceSwiftV2 = "kubernetes.azure.com/pod-network-instance"
19+
EnvPodCIDRs = "POD_CIDRs"
20+
EnvServiceCIDRs = "SERVICE_CIDRs"
21+
EnvInfraVNETCIDRs = "INFRA_VNET_CIDRs"
2122
)
2223

2324
// ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment.

‎cns/middlewares/k8sSwiftV2.go‎

Lines changed: 62 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -82,15 +82,18 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
8282
// Set routes for the pod
8383
for i := range ipConfigsResp.PodIPInfo {
8484
ipInfo := &ipConfigsResp.PodIPInfo[i]
85-
err = k.setRoutes(ipInfo)
86-
if err != nil {
87-
return &cns.IPConfigsResponse{
88-
Response: cns.Response{
89-
ReturnCode: types.FailedToAllocateIPConfig,
90-
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
91-
},
92-
PodIPInfo: []cns.PodIpInfo{},
93-
}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
85+
// Backend nics doesn't need routes to be set
86+
if ipInfo.NICType != cns.BackendNIC {
87+
err = k.setRoutes(ipInfo)
88+
if err != nil {
89+
return &cns.IPConfigsResponse{
90+
Response: cns.Response{
91+
ReturnCode: types.FailedToAllocateIPConfig,
92+
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
93+
},
94+
PodIPInfo: []cns.PodIpInfo{},
95+
}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
96+
}
9497
}
9598
}
9699
return ipConfigsResp, nil
@@ -115,8 +118,10 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
115118
}
116119

117120
// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
118-
if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok {
119-
req.SecondaryInterfacesExist = true
121+
_, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2]
122+
_, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2]
123+
if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel {
124+
120125
// Check if the MTPNC CRD exists for the pod, if not, return error
121126
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
122127
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
@@ -127,8 +132,27 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
127132
if !mtpnc.IsReady() {
128133
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
129134
}
135+
// If primary Ip is set in status field, it indicates the presence of secondary interfaces
136+
if mtpnc.Status.PrimaryIP != "" {
137+
req.SecondaryInterfacesExist = true
138+
}
139+
interfaceInfos := mtpnc.Status.InterfaceInfos
140+
for _, interfaceInfo := range interfaceInfos {
141+
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
142+
if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
143+
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
144+
}
145+
req.BackendInterfaceExist = true
146+
req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)
147+
148+
}
149+
if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC {
150+
req.SecondaryInterfacesExist = true
151+
}
152+
}
130153
}
131154
logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
155+
logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist)
132156
// retrieve podinfo from orchestrator context
133157
return podInfo, types.Success, ""
134158
}
@@ -171,19 +195,13 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
171195
// InterfaceName is empty for DelegatedVMNIC
172196
})
173197
} else {
174-
// Use InterfaceInfos if not empty
175-
podIPInfos = make([]cns.PodIpInfo, len(mtpnc.Status.InterfaceInfos))
176-
for i, interfaceInfo := range mtpnc.Status.InterfaceInfos {
177-
// Parse MTPNC primaryIP to get the IP address and prefix length
178-
ip, prefixSize, err := utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP)
179-
if err != nil {
180-
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
181-
}
182-
if prefixSize != prefixLength {
183-
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
184-
}
185-
186-
var nicType cns.NICType
198+
for _, interfaceInfo := range mtpnc.Status.InterfaceInfos {
199+
var (
200+
nicType cns.NICType
201+
ip string
202+
prefixSize int
203+
err error
204+
)
187205
switch {
188206
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && !interfaceInfo.AccelnetEnabled:
189207
nicType = cns.DelegatedVMNIC
@@ -194,19 +212,27 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
194212
default:
195213
nicType = cns.DelegatedVMNIC
196214
}
197-
198-
podIPInfos[i] = cns.PodIpInfo{
199-
PodIPConfig: cns.IPSubnet{
200-
IPAddress: ip,
201-
PrefixLength: uint8(prefixSize),
202-
},
203-
MacAddress: interfaceInfo.MacAddress,
204-
NICType: nicType,
205-
SkipDefaultRoutes: false,
206-
HostPrimaryIPInfo: cns.HostIPInfo{
207-
Gateway: interfaceInfo.GatewayIP,
208-
},
215+
if nicType != cns.NodeNetworkInterfaceBackendNIC {
216+
// Parse MTPNC primaryIP to get the IP address and prefix length
217+
ip, prefixSize, err = utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP)
218+
if err != nil {
219+
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
220+
}
221+
if prefixSize != prefixLength {
222+
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
223+
}
224+
podIPInfos = append(podIPInfos, cns.PodIpInfo{
225+
PodIPConfig: cns.IPSubnet{
226+
IPAddress: ip,
227+
PrefixLength: uint8(prefixSize),
228+
},
229+
MacAddress: interfaceInfo.MacAddress,
230+
NICType: cns.DelegatedVMNIC,
231+
SkipDefaultRoutes: false,
232+
// InterfaceName is empty for DelegatedVMNIC
233+
})
209234
}
235+
210236
}
211237
}
212238

‎cns/middlewares/k8sSwiftV2_linux_test.go‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ var (
3131

3232
testPod7GUID = "123e4567-e89b-12d3-a456-426614174000"
3333
testPod7Info = cns.NewPodInfo("123e45-eth0", testPod7GUID, "testpod7", "testpod7namespace")
34+
35+
testPod8GUID = "2006cad4-e54d-472e-863d-c4bac66200a7"
36+
testPod8Info = cns.NewPodInfo("2006cad4-eth0", testPod8GUID, "testpod8", "testpod8namespace")
37+
38+
testPod9GUID = "2006cad4-e54d-472e-863d-c4bac66200a7"
39+
testPod9Info = cns.NewPodInfo("2006cad4-eth0", testPod9GUID, "testpod9", "testpod9namespace")
3440
)
3541

3642
func TestMain(m *testing.M) {
@@ -141,6 +147,35 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
141147
assert.Equal(t, err, "")
142148
assert.Equal(t, respCode, types.Success)
143149
assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
150+
151+
happyReq2 := &cns.IPConfigsRequest{
152+
PodInterfaceID: testPod8Info.InterfaceID(),
153+
InfraContainerID: testPod8Info.InfraContainerID(),
154+
}
155+
156+
b, _ = testPod8Info.OrchestratorContext()
157+
happyReq2.OrchestratorContext = b
158+
happyReq2.SecondaryInterfacesExist = false
159+
160+
_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
161+
assert.Equal(t, err, "")
162+
assert.Equal(t, respCode, types.Success)
163+
assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
164+
165+
happyReq3 := &cns.IPConfigsRequest{
166+
PodInterfaceID: testPod9Info.InterfaceID(),
167+
InfraContainerID: testPod9Info.InfraContainerID(),
168+
}
169+
170+
b, _ = testPod9Info.OrchestratorContext()
171+
happyReq3.OrchestratorContext = b
172+
happyReq3.SecondaryInterfacesExist = false
173+
174+
_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
175+
assert.Equal(t, err, "")
176+
assert.Equal(t, respCode, types.Success)
177+
assert.Equal(t, happyReq3.SecondaryInterfacesExist, false)
178+
assert.Equal(t, happyReq3.BackendInterfaceExist, true)
144179
}
145180

146181
func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
@@ -373,11 +408,8 @@ func TestNICTypeConfigSuccess(t *testing.T) {
373408
if err != nil {
374409
t.Fatalf("unexpected error: %v", err)
375410
}
376-
if len(ipInfos) != 1 {
377-
t.Fatalf("expected 1 ipInfo, got %d", len(ipInfos))
378-
}
379-
if ipInfos[0].NICType != cns.BackendNIC {
380-
t.Errorf("expected NIC type %v, got %v", cns.BackendNIC, ipInfos[0].NICType)
411+
if len(ipInfos) != 0 {
412+
t.Fatalf("expected 0 ipInfo, got %d", len(ipInfos))
381413
}
382414
}
383415

@@ -404,7 +436,7 @@ func TestGetSWIFTv2IPConfigMultiInterfaceSuccess(t *testing.T) {
404436
assert.Equal(t, err, nil)
405437
// Ensure that the length of ipInfos matches the number of InterfaceInfos
406438
// Adjust this according to the test setup in mock client
407-
expectedInterfaceCount := 3
439+
expectedInterfaceCount := 2
408440
assert.Equal(t, len(ipInfos), expectedInterfaceCount)
409441

410442
for _, ipInfo := range ipInfos {

‎cns/middlewares/mock/mockClient.go‎

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,18 @@ func NewClient() *Client {
5454
testPod7.Labels = make(map[string]string)
5555
testPod7.Labels[configuration.LabelPodSwiftV2] = podNetwork
5656

57+
testPod8 := v1.Pod{}
58+
testPod8.Labels = make(map[string]string)
59+
testPod8.Labels[configuration.LabelPodSwiftV2] = podNetwork
60+
61+
testPod9 := v1.Pod{}
62+
testPod9.Labels = make(map[string]string)
63+
testPod9.Labels[configuration.LabelPodSwiftV2] = podNetwork
64+
65+
testPod10 := v1.Pod{}
66+
testPod10.Labels = make(map[string]string)
67+
testPod10.Labels[configuration.LabelPodNetworkInstanceSwiftV2] = podNetwork
68+
5769
testInterfaceInfos1 := v1alpha1.InterfaceInfo{
5870
NCID: "testncid",
5971
PrimaryIP: "192.168.0.1/32",
@@ -107,22 +119,72 @@ func NewClient() *Client {
107119
},
108120
}
109121

122+
testMTPNC8 := v1alpha1.MultitenantPodNetworkConfig{
123+
Status: v1alpha1.MultitenantPodNetworkConfigStatus{
124+
PrimaryIP: "192.168.0.1/32",
125+
MacAddress: "00:00:00:00:00:00",
126+
GatewayIP: "10.0.0.1",
127+
NCID: "testncid",
128+
InterfaceInfos: []v1alpha1.InterfaceInfo{
129+
{
130+
PrimaryIP: "192.168.0.1/32",
131+
MacAddress: "00:00:00:00:00:00",
132+
GatewayIP: "10.0.0.1",
133+
NCID: "testncid",
134+
DeviceType: v1alpha1.DeviceTypeVnetNIC,
135+
},
136+
{
137+
PrimaryIP: "192.168.0.1/32",
138+
MacAddress: "00:00:00:00:00:00",
139+
GatewayIP: "10.0.0.1",
140+
NCID: "testncid",
141+
DeviceType: v1alpha1.DeviceTypeInfiniBandNIC,
142+
},
143+
},
144+
},
145+
}
146+
// Mtpnc with just Infiniband interface
147+
testMTPNC9 := v1alpha1.MultitenantPodNetworkConfig{
148+
Status: v1alpha1.MultitenantPodNetworkConfigStatus{
149+
InterfaceInfos: []v1alpha1.InterfaceInfo{
150+
{
151+
PrimaryIP: "192.168.0.1/32",
152+
MacAddress: "00:00:00:00:00:00",
153+
GatewayIP: "10.0.0.1",
154+
NCID: "testncid",
155+
DeviceType: v1alpha1.DeviceTypeInfiniBandNIC,
156+
},
157+
},
158+
},
159+
}
160+
161+
// Mtpnc with just Infiniband interface
162+
testMTPNC10 := v1alpha1.MultitenantPodNetworkConfig{
163+
Status: v1alpha1.MultitenantPodNetworkConfigStatus{},
164+
}
165+
110166
return &Client{
111167
mtPodCache: map[string]*v1.Pod{
112-
"testpod1namespace/testpod1": &testPod1,
113-
"testpod3namespace/testpod3": &testPod3,
114-
"testpod4namespace/testpod4": &testPod4,
115-
"testpod5namespace/testpod5": &testPod5,
116-
"testpod6namespace/testpod6": &testPod6,
117-
"testpod7namespace/testpod7": &testPod7,
168+
"testpod1namespace/testpod1": &testPod1,
169+
"testpod3namespace/testpod3": &testPod3,
170+
"testpod4namespace/testpod4": &testPod4,
171+
"testpod5namespace/testpod5": &testPod5,
172+
"testpod6namespace/testpod6": &testPod6,
173+
"testpod7namespace/testpod7": &testPod7,
174+
"testpod8namespace/testpod8": &testPod8,
175+
"testpod9namespace/testpod9": &testPod9,
176+
"testpod10namespace/testpod10": &testPod10,
118177
},
119178
mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{
120-
"testpod1namespace/testpod1": &testMTPNC1,
121-
"testpod2namespace/testpod2": &testMTPNC2,
122-
"testpod4namespace/testpod4": &testMTPNC4,
123-
"testpod5namespace/testpod5": &testMTPNC3,
124-
"testpod6namespace/testpod6": &testMTPNC5,
125-
"testpod7namespace/testpod7": &testMTPNCMulti,
179+
"testpod1namespace/testpod1": &testMTPNC1,
180+
"testpod2namespace/testpod2": &testMTPNC2,
181+
"testpod4namespace/testpod4": &testMTPNC4,
182+
"testpod5namespace/testpod5": &testMTPNC3,
183+
"testpod6namespace/testpod6": &testMTPNC5,
184+
"testpod7namespace/testpod7": &testMTPNCMulti,
185+
"testpod8namespace/testpod8": &testMTPNC8,
186+
"testpod9namespace/testpod9": &testMTPNC9,
187+
"testpod10namespace/testpod10": &testMTPNC10,
126188
},
127189
}
128190
}

0 commit comments

Comments
 (0)