Skip to content

Commit ece0634

Browse files
sunya-chsunya-ch
authored
Merge pull request #331 from sunya-ch/feat-multi-config
feat: add multi-config and handle static ips
2 parents 8c66962 + 51775aa commit ece0634

File tree

5 files changed

+258
-6
lines changed

5 files changed

+258
-6
lines changed

‎cni/plugins/main/multi-nic/ipvlan.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/containernetworking/cni/pkg/types"
1313
current "github.com/containernetworking/cni/pkg/types/100"
14+
"github.com/containernetworking/plugins/pkg/utils"
1415
"github.com/vishvananda/netlink"
1516
)
1617

@@ -40,6 +41,17 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current
4041
return
4142
}
4243

44+
var ipamMap map[string][]byte
45+
46+
if n.IPAM.Type == MultiConfigIPAMType {
47+
var err error
48+
ipamMap, err = getMultiIPAMConfigBytes(bytes)
49+
if err != nil {
50+
ipamMap = nil
51+
utils.Logger.Debug(fmt.Sprintf("getMultiIPAMConfigBytes failed: %v", err))
52+
}
53+
}
54+
4355
// interfaces are orderly assigned from interface set
4456
for index, masterName := range n.Masters {
4557
if masterName == "" {
@@ -62,9 +74,16 @@ func loadIPVANConf(bytes []byte, ifName string, n *NetConf, ipConfigs []*current
6274
return
6375
}
6476

65-
if n.IsMultiNICIPAM {
66-
// multi-NIC IPAM config
77+
if len(ipConfigs) > 0 {
78+
// no need to call ipam due to static ip
6779
confBytes, multiPathRoutes = injectMultiNicIPAM(confBytes, bytes, ipConfigs, index)
80+
} else if ipamMap != nil {
81+
if ipamBytes, found := ipamMap[masterName]; found {
82+
confBytes, multiPathRoutes = replaceSingleNicIPAMWithMultiConfig(confBytes, bytes, ipamBytes)
83+
} else {
84+
utils.Logger.Debug(fmt.Sprintf("Multi-config IPAM has no definition of %s", masterName))
85+
continue
86+
}
6887
} else {
6988
confBytes, multiPathRoutes = injectSingleNicIPAM(confBytes, bytes)
7089
}

‎cni/plugins/main/multi-nic/multi-nic.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,30 @@ func cmdAdd(args *skel.CmdArgs) error {
125125
if len(result.IPs) == 0 {
126126
return fmt.Errorf("IPAM plugin returned missing IP config %v", string(injectedStdIn))
127127
}
128+
} else if !haveResult && n.IPAM.Type == MultiConfigIPAMType {
129+
var ips []string
130+
ips, args.Args = getStaticIPs(args.Args)
131+
for index, ipnet := range ips {
132+
ipVal, reservedIP, err := net.ParseCIDR(ipnet)
133+
if err != nil {
134+
utils.Logger.Debug(fmt.Sprintf("failed to parse static IP %s: %v", ipnet, err))
135+
return err
136+
}
137+
reservedIP.IP = ipVal
138+
ipConf := &current.IPConfig{
139+
Address: *reservedIP,
140+
Interface: current.Int(index),
141+
}
142+
result.IPs = append(result.IPs, ipConf)
143+
}
144+
if len(n.Masters) == 0 {
145+
ipamObject, err := getMultiIPAMConfig(args.StdinData)
146+
if err == nil && ipamObject.IPAM.Args != nil {
147+
for masterName, _ := range ipamObject.IPAM.Args {
148+
n.Masters = append(n.Masters, masterName)
149+
}
150+
}
151+
}
128152
}
129153

130154
// get device config and apply

‎cni/plugins/main/multi-nic/multi-nic_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -533,6 +533,71 @@ var _ = Describe("Operations", func() {
533533
multiPathTest(ver, singleNICIPAMWithMultiPath, fullNets, POOL_MASTER_NAMES)
534534
})
535535

536+
It(fmt.Sprintf("[%s] check multi-config IPAM", ver), func() {
537+
conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets)
538+
confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, []*types100.IPConfig{})
539+
Expect(err).NotTo(HaveOccurred())
540+
Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES)))
541+
for index, confBytes := range confBytesArray {
542+
log.Printf("%s", confBytes)
543+
confObj := &IPVLANTypeNetConf{}
544+
err = json.Unmarshal(confBytes, confObj)
545+
Expect(err).NotTo(HaveOccurred())
546+
Expect(confObj.IPAM.Type).To(Equal("whereabouts"))
547+
ipamObject := &IPAMExtract{}
548+
err = json.Unmarshal(confBytes, ipamObject)
549+
Expect(err).NotTo(HaveOccurred())
550+
Expect(ipamObject.IPAM["range"].(string)).To(Equal(POOL_IP_ADDRESSES[index]))
551+
Expect(ipamObject.IPAM["network_name"].(string)).To(Equal(POOL_MASTER_NAMES[index]))
552+
}
553+
})
554+
555+
It(fmt.Sprintf("[%s] check multi-config IPAM with static IP", ver), func() {
556+
var ips []*types100.IPConfig
557+
for _, podIP := range POOL_IP_ADDRESSES {
558+
ipVal, ipnet, err := net.ParseCIDR(podIP)
559+
ipnet.IP = ipVal
560+
Expect(err).NotTo(HaveOccurred())
561+
podIPConfig := &types100.IPConfig{Address: *ipnet}
562+
ips = append(ips, podIPConfig)
563+
}
564+
conf, n := getSampleMultiIPAMConfig(ver, POOL_MASTER_NAMES, masterNets)
565+
confBytesArray, _, err := loadIPVANConf([]byte(conf), "net1", n, ips)
566+
Expect(err).NotTo(HaveOccurred())
567+
Expect(len(confBytesArray)).To(Equal(len(POOL_MASTER_NAMES)))
568+
for index, confBytes := range confBytesArray {
569+
log.Printf("%s", confBytes)
570+
confObj := &IPVLANTypeNetConf{}
571+
err = json.Unmarshal(confBytes, confObj)
572+
Expect(err).NotTo(HaveOccurred())
573+
Expect(confObj.IPAM.Type).To(Equal("static"))
574+
ipamObject := &IPAMExtract{}
575+
err = json.Unmarshal(confBytes, ipamObject)
576+
Expect(err).NotTo(HaveOccurred())
577+
addresses := ipamObject.IPAM["addresses"].([]interface{})
578+
Expect(len(addresses)).To(Equal(1))
579+
addressMap := addresses[0].(map[string]interface{})
580+
Expect(addressMap["address"].(string)).To(Equal(POOL_IP_ADDRESSES[index]))
581+
}
582+
})
583+
584+
It(fmt.Sprintf("[%s] check getStaticIPs", ver), func() {
585+
var ips []string
586+
joinedIPs := strings.Join(POOL_IP_ADDRESSES, ",")
587+
argStr := fmt.Sprintf("POD_NAME=a;IP=%s", joinedIPs)
588+
ips, argStr = getStaticIPs(argStr)
589+
for index, podIP := range ips {
590+
Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index]))
591+
}
592+
Expect(argStr).To(Equal("POD_NAME=a"))
593+
594+
argStr = fmt.Sprintf("POD_NAME=a;IP=%s;SOME_ARG=b", joinedIPs)
595+
ips, argStr = getStaticIPs(argStr)
596+
for index, podIP := range ips {
597+
Expect(podIP).To(Equal(POOL_IP_ADDRESSES[index]))
598+
}
599+
Expect(argStr).To(Equal("POD_NAME=a;SOME_ARG=b"))
600+
})
536601
}
537602
})
538603

@@ -583,6 +648,48 @@ func getAwsIpvlanConfig(ver, masterNets string) ([]byte, *NetConf) {
583648
return conf, n
584649
}
585650

651+
func getSampleMultiIPAMConfig(ver string, masterNames []string, masterNets string) ([]byte, *NetConf) {
652+
ipamArgs := ""
653+
for index, masterName := range masterNames {
654+
if index > 0 {
655+
ipamArgs += ","
656+
}
657+
ipamArgs += fmt.Sprintf(`
658+
"%s": {"range": "%s"}
659+
`, masterName, POOL_IP_ADDRESSES[index])
660+
}
661+
confStr := fmt.Sprintf(`{
662+
"cniVersion": "%s",
663+
"name": "multi-nic-sample",
664+
"type": "multi-nic",
665+
"plugin": {
666+
"cniVersion": "0.3.0",
667+
"type": "ipvlan",
668+
"mode": "l2"
669+
},
670+
"vlanMode": "l2",
671+
"ipam": {
672+
"type": "multi-config",
673+
"ipam_type": "whereabouts",
674+
"args": {
675+
%s
676+
}
677+
},
678+
"daemonIP": "%s",
679+
"daemonPort": %d,
680+
"subnet": "192.168.0.0/16",
681+
"masterNets": %s
682+
}`, ver, ipamArgs, BRIDGE_HOST_IP, daemonPort, masterNets)
683+
log.Printf("%s", confStr)
684+
conf := []byte(confStr)
685+
n := &NetConf{}
686+
err := json.Unmarshal(conf, n)
687+
Expect(err).NotTo(HaveOccurred())
688+
n.DeviceIDs = POOL_MASTER_NAMES
689+
n.Masters = POOL_MASTER_NAMES
690+
return conf, n
691+
}
692+
586693
func closeServer(srv *http.Server, httpServerExitDone *sync.WaitGroup) {
587694
if err := srv.Shutdown(context.TODO()); err != nil {
588695
panic(err)

‎cni/plugins/main/multi-nic/utils.go‎

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,27 @@ import (
2020
"github.com/vishvananda/netlink"
2121
)
2222

23+
const (
24+
MultiConfigIPAMType = "multi-config"
25+
WhereaboutsIPAMType = "whereabouts"
26+
)
27+
28+
type IPAMExtract struct {
29+
IPAM map[string]interface{} `json:"ipam"`
30+
}
31+
32+
type MultiIPAMConfig struct {
33+
Name string
34+
Type string `json:"type"`
35+
IpamType string `json:"ipam_type"`
36+
Args map[string]map[string]interface{} `json:"args"`
37+
Routes []*types.Route `json:"routes"`
38+
}
39+
40+
type MultiIPAMExtract struct {
41+
IPAM MultiIPAMConfig `json:"ipam"`
42+
}
43+
2344
// getPodInfo extracts pod Name and Namespace from cniArgs
2445
func getPodInfo(cniArgs string) (string, string) {
2546
splits := strings.Split(cniArgs, ";")
@@ -35,6 +56,57 @@ func getPodInfo(cniArgs string) (string, string) {
3556
return podName, podNamespace
3657
}
3758

59+
// getStaticIPs extracts static ip from cniArgs
60+
// reference: https://github.com/k8snetworkplumbingwg/multus-cni/blob/e2e8cfb677e8cf5352f737b5004effed7518d71a/pkg/multus/multus.go#L324
61+
func getStaticIPs(cniArgs string) ([]string, string) {
62+
splits := strings.Split(cniArgs, ";")
63+
for index, split := range splits {
64+
if strings.HasPrefix(split, "IP=") {
65+
ipsStr := strings.TrimPrefix(split, "IP=")
66+
newItems := splits[0:index]
67+
if index < len(split)-1 {
68+
newItems = append(newItems, splits[index+1:len(splits)]...)
69+
}
70+
newCniArgs := strings.Join(newItems, ";")
71+
return strings.Split(ipsStr, ","), newCniArgs
72+
}
73+
}
74+
return []string{}, cniArgs
75+
}
76+
77+
func getMultiIPAMConfig(multiNicConfBytes []byte) (*MultiIPAMExtract, error) {
78+
ipamObject := &MultiIPAMExtract{}
79+
err := json.Unmarshal(multiNicConfBytes, ipamObject)
80+
return ipamObject, err
81+
82+
}
83+
84+
func getMultiIPAMConfigBytes(multiNicConfBytes []byte) (map[string][]byte, error) {
85+
multiIPAMConfigBytes := make(map[string][]byte)
86+
ipamObject, err := getMultiIPAMConfig(multiNicConfBytes)
87+
if err == nil && ipamObject.IPAM.Args != nil {
88+
for masterName, singleIPAMObject := range ipamObject.IPAM.Args {
89+
singleIPAMConfBytes := make(map[string]interface{})
90+
if singleIPAMObject == nil {
91+
utils.Logger.Debug(fmt.Sprintf("ipamObject.IPAM.Args not defined on %s", masterName))
92+
continue
93+
}
94+
// set type
95+
singleIPAMConfBytes["type"] = ipamObject.IPAM.IpamType
96+
if ipamObject.IPAM.IpamType == WhereaboutsIPAMType {
97+
singleIPAMConfBytes["network_name"] = masterName
98+
}
99+
// set args
100+
for k, v := range singleIPAMObject {
101+
singleIPAMConfBytes[k] = v
102+
}
103+
ipamBytes, _ := json.Marshal(singleIPAMConfBytes)
104+
multiIPAMConfigBytes[masterName] = ipamBytes
105+
}
106+
}
107+
return multiIPAMConfigBytes, err
108+
}
109+
38110
// injectIPAM injects ipam bytes to config
39111
func injectMultiNicIPAM(singleNicConfBytes, multiNicConfBytes []byte, ipConfigs []*current.IPConfig, ipIndex int) ([]byte, map[string][]*netlink.NexthopInfo) {
40112
var ipConfig *current.IPConfig
@@ -48,11 +120,17 @@ func injectSingleNicIPAM(singleNicConfBytes []byte, multiNicConfBytes []byte) ([
48120
return replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes)
49121
}
50122

51-
type IPAMExtract struct {
52-
IPAM map[string]interface{} `json:"ipam"`
123+
func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
124+
ipamObject := &IPAMExtract{}
125+
err := json.Unmarshal(multiNicConfBytes, ipamObject)
126+
if err == nil {
127+
ipamBytes, _ := json.Marshal(ipamObject.IPAM)
128+
return replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes)
129+
}
130+
return singleNicConfBytes, nil
53131
}
54132

55-
func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
133+
func replaceSingleNicIPAMWithMultiConfig(singleNicConfBytes, multiNicConfBytes, ipamBytes []byte) ([]byte, map[string][]*netlink.NexthopInfo) {
56134
confStr := string(singleNicConfBytes)
57135
ipamObject := &IPAMExtract{}
58136
err := json.Unmarshal(multiNicConfBytes, ipamObject)
@@ -62,7 +140,6 @@ func replaceSingleNicIPAM(singleNicConfBytes, multiNicConfBytes []byte) ([]byte,
62140
ipamObject.IPAM["routes"] = nonMultiPathRoutes
63141
}
64142

65-
ipamBytes, _ := json.Marshal(ipamObject.IPAM)
66143
singleIPAM := fmt.Sprintf("\"ipam\":%s", string(ipamBytes))
67144
injectedStr := strings.ReplaceAll(confStr, "\"ipam\":{}", singleIPAM)
68145
return []byte(injectedStr), multiPathRoutes

‎config/samples/multinicnetwork/ipvlanl2_multiconfig.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: multinic.fms.io/v1
2+
kind: MultiNicNetwork
3+
metadata:
4+
name: multinic-multiconfig
5+
spec:
6+
subnet: ""
7+
ipam: |
8+
{
9+
"type": "multi-config",
10+
"ipam_type": "whereabouts",
11+
"args": {
12+
"p0": {
13+
"range": "192.168.0.0/18"
14+
},
15+
"p1": {
16+
"range": "192.168.64.0/18"
17+
}
18+
}
19+
}
20+
multiNICIPAM: false
21+
plugin:
22+
cniVersion: "0.3.0"
23+
type: ipvlan
24+
args:
25+
mode: l2

0 commit comments

Comments
 (0)