Skip to content

Commit ee19cc4

Browse files
BSWANGBSWANG
committed
ENO api fallback to hc-eni-host path
Signed-off-by: bingshen.wbs <[email protected]>
1 parent c1afe28 commit ee19cc4

File tree

13 files changed

+377
-254
lines changed

13 files changed

+377
-254
lines changed

‎pkg/eni/remote.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"github.com/go-logr/logr"
99
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1010
k8stypes "k8s.io/apimachinery/pkg/types"
11-
"k8s.io/utils/ptr"
1211
"sigs.k8s.io/controller-runtime/pkg/client"
1312
logf "sigs.k8s.io/controller-runtime/pkg/log"
1413

@@ -82,11 +81,6 @@ func (l *RemoteIPResource) ToRPC() []*rpc.NetConf {
8281

8382
eniInfo.VfId = info.VfID
8483

85-
switch l.podENI.Annotations[types.ENOApi] {
86-
case types.APIEcsHDeni:
87-
eniInfo.VfType = ptr.To(rpc.VfType_VfTypeVPC)
88-
}
89-
9084
netConf = append(netConf, &rpc.NetConf{
9185
BasicInfo: &rpc.BasicInfo{
9286
PodIP: podIP,

‎pkg/eni/remote_test.go‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"k8s.io/apimachinery/pkg/runtime"
1313
"sigs.k8s.io/controller-runtime/pkg/client/fake"
1414

15-
"github.com/AliyunContainerService/terway/rpc"
1615
"github.com/AliyunContainerService/terway/types"
1716
"github.com/AliyunContainerService/terway/types/daemon"
1817

@@ -74,14 +73,10 @@ func TestToRPC(t *testing.T) {
7473
assert.Equal(t, true, result[0].DefaultRoute)
7574
})
7675

77-
t.Run("test with VfTypeVPC when APIEcsHDeni annotation is set", func(t *testing.T) {
76+
t.Run("test with VfId set in ENIInfo", func(t *testing.T) {
77+
vfID := uint32(5)
7878
l := &RemoteIPResource{
7979
podENI: networkv1beta1.PodENI{
80-
ObjectMeta: metav1.ObjectMeta{
81-
Annotations: map[string]string{
82-
types.ENOApi: types.APIEcsHDeni,
83-
},
84-
},
8580
Spec: networkv1beta1.PodENISpec{
8681
Allocations: []networkv1beta1.Allocation{
8782
{
@@ -98,7 +93,9 @@ func TestToRPC(t *testing.T) {
9893
},
9994
Status: networkv1beta1.PodENIStatus{
10095
ENIInfos: map[string]networkv1beta1.ENIInfo{
101-
"eni-11": {},
96+
"eni-11": {
97+
VfID: &vfID,
98+
},
10299
},
103100
},
104101
},
@@ -107,15 +104,13 @@ func TestToRPC(t *testing.T) {
107104
result := l.ToRPC()
108105
assert.NotNil(t, result)
109106
assert.Equal(t, 1, len(result))
110-
assert.Equal(t, rpc.VfType_VfTypeVPC, *result[0].ENIInfo.VfType)
107+
assert.NotNil(t, result[0].ENIInfo.VfId)
108+
assert.Equal(t, uint32(5), *result[0].ENIInfo.VfId)
111109
})
112110

113-
t.Run("test without ENOApi annotation", func(t *testing.T) {
111+
t.Run("test with VfId nil in ENIInfo", func(t *testing.T) {
114112
l := &RemoteIPResource{
115113
podENI: networkv1beta1.PodENI{
116-
ObjectMeta: metav1.ObjectMeta{
117-
Annotations: map[string]string{},
118-
},
119114
Spec: networkv1beta1.PodENISpec{
120115
Allocations: []networkv1beta1.Allocation{
121116
{
@@ -132,7 +127,9 @@ func TestToRPC(t *testing.T) {
132127
},
133128
Status: networkv1beta1.PodENIStatus{
134129
ENIInfos: map[string]networkv1beta1.ENIInfo{
135-
"eni-11": {},
130+
"eni-11": {
131+
VfID: nil,
132+
},
136133
},
137134
},
138135
},
@@ -141,8 +138,9 @@ func TestToRPC(t *testing.T) {
141138
result := l.ToRPC()
142139
assert.NotNil(t, result)
143140
assert.Equal(t, 1, len(result))
144-
assert.Nil(t, result[0].ENIInfo.VfType)
141+
assert.Nil(t, result[0].ENIInfo.VfId)
145142
})
143+
146144
}
147145

148146
func TestAllocateReturnsErrorWhenResourceTypeMismatch(t *testing.T) {

‎plugin/driver/vf/vf.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@ import (
1313
)
1414

1515
const (
16-
defaultNUSAConfigPath = "/var/rdma/eni_topo"
17-
defaultSysfsBasePath = "/sys/bus/pci/devices"
16+
defaultSysfsBasePath = "/sys/bus/pci/devices"
17+
vfBind = "/sys/bus/pci/drivers/virtio-pci"
18+
)
1819

19-
vfBind = "/sys/bus/pci/drivers/virtio-pci"
20+
var (
21+
defaultNUSAConfigPath = "/var/rdma/eni_topo"
22+
HcENIHostConfigPath = "/var/run/hc-eni-host/vf-topo-vpc"
2023
)
2124

2225
type Config struct {
@@ -48,7 +51,14 @@ func parse(path string, config []byte) (*Configs, error) {
4851

4952
func GetBDFbyVFID(path string, vfID int) (string, error) {
5053
if path == "" {
51-
path = defaultNUSAConfigPath
54+
_, err := os.Stat(defaultNUSAConfigPath)
55+
if err == nil {
56+
path = defaultNUSAConfigPath
57+
} else if os.IsNotExist(err) {
58+
path = HcENIHostConfigPath
59+
} else {
60+
return "", err
61+
}
5262
}
5363
configContent, err := os.ReadFile(path)
5464
if err != nil {

‎plugin/driver/vf/vf_test.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,182 @@ func TestParse(t *testing.T) {
4040
})
4141
}
4242

43+
// TestGetBDFbyVFID tests the GetBDFbyVFID function
44+
func TestGetBDFbyVFID(t *testing.T) {
45+
t.Run("valid config with matching vfID", func(t *testing.T) {
46+
// Create a temporary directory and config file
47+
tempDir := t.TempDir()
48+
configPath := filepath.Join(tempDir, "vf-config.json")
49+
50+
// Write test config data
51+
configData := `{"eniVFs": [{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}]}`
52+
err := os.WriteFile(configPath, []byte(configData), 0644)
53+
require.NoError(t, err)
54+
55+
// Test the function
56+
bdf, err := GetBDFbyVFID(configPath, 9)
57+
assert.NoError(t, err)
58+
assert.Equal(t, "0000:1:1.6", bdf)
59+
})
60+
61+
t.Run("valid config with non-matching vfID", func(t *testing.T) {
62+
// Create a temporary directory and config file
63+
tempDir := t.TempDir()
64+
configPath := filepath.Join(tempDir, "vf-config.json")
65+
66+
// Write test config data
67+
configData := `{"eniVFs": [{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}]}`
68+
err := os.WriteFile(configPath, []byte(configData), 0644)
69+
require.NoError(t, err)
70+
71+
// Test the function with non-matching vfID
72+
_, err = GetBDFbyVFID(configPath, 10)
73+
assert.Error(t, err)
74+
assert.Contains(t, err.Error(), "not found specified vfID 10")
75+
})
76+
77+
t.Run("invalid config file", func(t *testing.T) {
78+
// Create a temporary directory and config file
79+
tempDir := t.TempDir()
80+
configPath := filepath.Join(tempDir, "vf-config.json")
81+
82+
// Write invalid config data
83+
configData := `{"eniVFs": [{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}`
84+
err := os.WriteFile(configPath, []byte(configData), 0644)
85+
require.NoError(t, err)
86+
87+
// Test the function
88+
_, err = GetBDFbyVFID(configPath, 9)
89+
assert.Error(t, err)
90+
})
91+
92+
t.Run("non-existent config file", func(t *testing.T) {
93+
// Test the function with non-existent config file
94+
_, err := GetBDFbyVFID("/non/existent/path", 9)
95+
assert.Error(t, err)
96+
})
97+
98+
t.Run("empty path with defaultNUSAConfigPath existing", func(t *testing.T) {
99+
// Create a temporary directory and config file
100+
tempDir := t.TempDir()
101+
102+
// Save original value
103+
originalPath := defaultNUSAConfigPath
104+
// Temporarily change the default path to our temp directory
105+
defaultNUSAConfigPath = filepath.Join(tempDir, "eni_topo")
106+
107+
// Make sure the default path doesn't exist
108+
_, err := os.Stat(defaultNUSAConfigPath)
109+
if !os.IsNotExist(err) {
110+
os.Remove(defaultNUSAConfigPath)
111+
}
112+
113+
// Write test config data to the default path
114+
configData := `[{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}]`
115+
err = os.WriteFile(defaultNUSAConfigPath, []byte(configData), 0644)
116+
require.NoError(t, err)
117+
118+
// Test the function with empty path
119+
bdf, err := GetBDFbyVFID("", 9)
120+
assert.NoError(t, err)
121+
assert.Equal(t, "0000:1:1.6", bdf)
122+
123+
// Restore original value
124+
defaultNUSAConfigPath = originalPath
125+
})
126+
127+
t.Run("empty path with HcENIHostConfigPath existing", func(t *testing.T) {
128+
// Create a temporary directory and config file
129+
tempDir := t.TempDir()
130+
131+
// Save original values
132+
originalNUSAPath := defaultNUSAConfigPath
133+
originalHCPath := HcENIHostConfigPath
134+
135+
// Temporarily change the paths to our temp directory
136+
defaultNUSAConfigPath = filepath.Join(tempDir, "eni_topo")
137+
HcENIHostConfigPath = filepath.Join(tempDir, "vf-topo-vpc")
138+
139+
// Make sure the defaultNUSAConfigPath doesn't exist
140+
_, err := os.Stat(defaultNUSAConfigPath)
141+
if !os.IsNotExist(err) {
142+
os.Remove(defaultNUSAConfigPath)
143+
}
144+
145+
// Write test config data to the HcENIHostConfigPath
146+
configData := `{"eniVFs": [{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}]}`
147+
err = os.WriteFile(HcENIHostConfigPath, []byte(configData), 0644)
148+
require.NoError(t, err)
149+
150+
// Test the function with empty path
151+
bdf, err := GetBDFbyVFID("", 9)
152+
assert.NoError(t, err)
153+
assert.Equal(t, "0000:1:1.6", bdf)
154+
155+
// Restore original values
156+
defaultNUSAConfigPath = originalNUSAPath
157+
HcENIHostConfigPath = originalHCPath
158+
})
159+
160+
t.Run("empty path with neither config file existing", func(t *testing.T) {
161+
// Save original values
162+
originalNUSAPath := defaultNUSAConfigPath
163+
originalHCPath := HcENIHostConfigPath
164+
165+
// Temporarily change the paths to non-existent files
166+
tempDir := t.TempDir()
167+
defaultNUSAConfigPath = filepath.Join(tempDir, "non-existent-eni_topo")
168+
HcENIHostConfigPath = filepath.Join(tempDir, "non-existent-vf-topo-vpc")
169+
170+
// Test the function with empty path
171+
_, err := GetBDFbyVFID("", 9)
172+
assert.Error(t, err)
173+
174+
// Restore original values
175+
defaultNUSAConfigPath = originalNUSAPath
176+
HcENIHostConfigPath = originalHCPath
177+
})
178+
179+
t.Run("multiple VFs with matching vfID", func(t *testing.T) {
180+
// Create a temporary directory and config file
181+
tempDir := t.TempDir()
182+
configPath := filepath.Join(tempDir, "vf-config.json")
183+
184+
// Write test config data with multiple VFs
185+
configData := `{"eniVFs": [{"pf_id": 1,"vf_id": 8,"bdf": "0000:1:1.5"}, {"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}, {"pf_id": 1,"vf_id": 10,"bdf": "0000:1:1.7"}]}`
186+
err := os.WriteFile(configPath, []byte(configData), 0644)
187+
require.NoError(t, err)
188+
189+
// Test the function
190+
bdf, err := GetBDFbyVFID(configPath, 9)
191+
assert.NoError(t, err)
192+
assert.Equal(t, "0000:1:1.6", bdf)
193+
})
194+
195+
t.Run("eni controller config format", func(t *testing.T) {
196+
// Create a temporary directory and config file
197+
tempDir := t.TempDir()
198+
configPath := filepath.Join(tempDir, "eni-controller-config.json")
199+
200+
// Write test config data in eni controller format
201+
configData := `[{"pf_id": 1,"vf_id": 9,"bdf": "0000:1:1.6"}]`
202+
err := os.WriteFile(configPath, []byte(configData), 0644)
203+
require.NoError(t, err)
204+
205+
// Temporarily change the default path to test eni controller format
206+
originalPath := defaultNUSAConfigPath
207+
defaultNUSAConfigPath = configPath
208+
209+
// Test the function
210+
bdf, err := GetBDFbyVFID(configPath, 9)
211+
assert.NoError(t, err)
212+
assert.Equal(t, "0000:1:1.6", bdf)
213+
214+
// Restore original value
215+
defaultNUSAConfigPath = originalPath
216+
})
217+
}
218+
43219
// TestGetPFBDF tests the getPFBDF function with mocked sysfs structure
44220
func TestGetPFBDF(t *testing.T) {
45221
t.Run("invalid BDF format", func(t *testing.T) {

‎plugin/terway/cni.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -290,14 +290,9 @@ func parseSetupConf(ctx context.Context, args *skel.CmdArgs, alloc *rpc.NetConf,
290290
if alloc.GetENIInfo() != nil {
291291
mac := alloc.GetENIInfo().GetMAC()
292292
vfID = alloc.GetENIInfo().VfId
293-
vfType := rpc.VfType_VfTypeDefault
294293
if vfID != nil {
295294
// when do setup, this link must present
296-
if alloc.GetENIInfo().VfType != nil {
297-
vfType = *alloc.GetENIInfo().VfType
298-
}
299-
300-
deviceID, err = prepareVF(ctx, int(*vfID), mac, vfType)
295+
deviceID, err = prepareVF(ctx, int(*vfID), mac)
301296
if err != nil {
302297
return nil, err
303298
}

‎plugin/terway/cni_linux.go‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -466,19 +466,8 @@ func doCmdCheck(ctx context.Context, client rpc.TerwayBackendClient, cmdArgs *cn
466466
return nil
467467
}
468468

469-
func prepareVF(ctx context.Context, id int, mac string, vfType rpc.VfType) (int32, error) {
470-
// vf-topo-vpc
471-
configPath := ""
472-
473-
switch vfType {
474-
case rpc.VfType_VfTypeDefault:
475-
case rpc.VfType_VfTypeVPC:
476-
configPath = "/var/run/hc-eni-host/vf-topo-vpc"
477-
default:
478-
return 0, fmt.Errorf("not support this vf type")
479-
}
480-
481-
deviceID, err := vf.SetupDriverAndGetNetInterface(ctx, id, configPath)
469+
func prepareVF(ctx context.Context, id int, mac string) (int32, error) {
470+
deviceID, err := vf.SetupDriverAndGetNetInterface(ctx, id, "")
482471
if err != nil {
483472
return 0, err
484473
}

‎plugin/terway/cni_linux_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ func TestDoCmdAdd(t *testing.T) {
398398
})
399399

400400
// Mock prepareVF function
401-
patches.ApplyFunc(prepareVF, func(ctx context.Context, id int, mac string, vfType rpc.VfType) (int32, error) {
401+
patches.ApplyFunc(prepareVF, func(ctx context.Context, id int, mac string) (int32, error) {
402402
return 1, nil
403403
})
404404

0 commit comments

Comments
 (0)