Skip to content

Commit bfb3eaa

Browse files
ashvindeodharYongli Chen
ashvindeodhar
authored and
Yongli Chen
committed
Implement multitenancy for windows (#259)
* Implement multitenancy for windows
1 parent 6e6260a commit bfb3eaa

File tree

10 files changed

+254
-57
lines changed

10 files changed

+254
-57
lines changed

‎cni/network/network.go‎

Lines changed: 69 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,31 @@ func GetEndpointID(args *cniSkel.CmdArgs) string {
131131
return infraEpId
132132
}
133133

134+
// getPodInfo returns POD info by parsing the CNI args.
135+
func (plugin *netPlugin) getPodInfo(args string) (string, string, error) {
136+
podCfg, err := cni.ParseCniArgs(args)
137+
if err != nil {
138+
log.Printf("Error while parsing CNI Args %v", err)
139+
return "", "", err
140+
}
141+
142+
k8sNamespace := string(podCfg.K8S_POD_NAMESPACE)
143+
if len(k8sNamespace) == 0 {
144+
errMsg := "Pod Namespace not specified in CNI Args"
145+
log.Printf(errMsg)
146+
return "", "", plugin.Errorf(errMsg)
147+
}
148+
149+
k8sPodName := string(podCfg.K8S_POD_NAME)
150+
if len(k8sPodName) == 0 {
151+
errMsg := "Pod Name not specified in CNI Args"
152+
log.Printf(errMsg)
153+
return "", "", plugin.Errorf(errMsg)
154+
}
155+
156+
return k8sPodName, k8sNamespace, nil
157+
}
158+
134159
//
135160
// CNI implementation
136161
// https://github.com/containernetworking/cni/blob/master/SPEC.md
@@ -192,26 +217,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
192217
}()
193218

194219
// Parse Pod arguments.
195-
podCfg, err := cni.ParseCniArgs(args.Args)
220+
k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
196221
if err != nil {
197-
log.Printf("Error while parsing CNI Args %v", err)
198222
return err
199223
}
200224

201-
k8sNamespace := string(podCfg.K8S_POD_NAMESPACE)
202-
if len(k8sNamespace) == 0 {
203-
errMsg := "Pod Namespace not specified in CNI Args"
204-
log.Printf(errMsg)
205-
return plugin.Errorf(errMsg)
206-
}
207-
208-
k8sPodName := string(podCfg.K8S_POD_NAME)
209-
if len(k8sPodName) == 0 {
210-
errMsg := "Pod Name not specified in CNI Args"
211-
log.Printf(errMsg)
212-
return plugin.Errorf(errMsg)
213-
}
214-
215225
k8sContainerID := args.ContainerID
216226
if len(k8sContainerID) == 0 {
217227
errMsg := "Container ID not specified in CNI Args"
@@ -234,10 +244,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
234244
}
235245
}
236246

237-
// Initialize values from network config.
238-
networkId := nwCfg.Name
239-
endpointId := GetEndpointID(args)
240-
241247
result, cnsNetworkConfig, subnetPrefix, azIpamResult, err = GetMultiTenancyCNIResult(enableInfraVnet, nwCfg, plugin, k8sPodName, k8sNamespace, args.IfName)
242248
if err != nil {
243249
log.Printf("GetMultiTenancyCNIResult failed with error %v", err)
@@ -252,6 +258,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
252258

253259
log.Printf("Result from multitenancy %+v", result)
254260

261+
// Initialize values from network config.
262+
networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
263+
if err != nil {
264+
log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
265+
return err
266+
}
267+
268+
endpointId := GetEndpointID(args)
269+
255270
policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs)
256271

257272
// Check whether the network already exists.
@@ -265,13 +280,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
265280
*/
266281
epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId)
267282
if epInfo != nil {
268-
result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg)
269-
if err != nil {
270-
log.Printf("handleConsecutiveAdd failed with error %v", err)
271-
return err
283+
resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg)
284+
if errConsAdd != nil {
285+
log.Printf("handleConsecutiveAdd failed with error %v", errConsAdd)
286+
result = resultConsAdd
287+
return errConsAdd
272288
}
273289

274-
if result != nil {
290+
if resultConsAdd != nil {
291+
result = resultConsAdd
275292
return nil
276293
}
277294
}
@@ -335,11 +352,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
335352
}
336353

337354
log.Printf("[cni-net] nwDNSInfo: %v", nwDNSInfo)
355+
// Update subnet prefix for multi-tenant scenario
356+
updateSubnetPrefix(cnsNetworkConfig, &subnetPrefix)
338357

339358
// Create the network.
340359
nwInfo := network.NetworkInfo{
341-
Id: networkId,
342-
Mode: nwCfg.Mode,
360+
Id: networkId,
361+
Mode: nwCfg.Mode,
362+
MasterIfName: masterIfName,
343363
Subnets: []network.SubnetInfo{
344364
network.SubnetInfo{
345365
Family: platform.AfINET,
@@ -492,8 +512,18 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error {
492512

493513
log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
494514

515+
// Parse Pod arguments.
516+
k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
517+
if err != nil {
518+
return err
519+
}
520+
495521
// Initialize values from network config.
496-
networkId := nwCfg.Name
522+
networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
523+
if err != nil {
524+
log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
525+
}
526+
497527
endpointId := GetEndpointID(args)
498528

499529
// Query the network.
@@ -552,8 +582,18 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
552582

553583
log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
554584

585+
// Parse Pod arguments.
586+
k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
587+
if err != nil {
588+
return err
589+
}
590+
555591
// Initialize values from network config.
556-
networkId := nwCfg.Name
592+
networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
593+
if err != nil {
594+
log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
595+
}
596+
557597
endpointId := GetEndpointID(args)
558598

559599
// Query the network.

‎cni/network/network_linux.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,10 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul
108108
func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy {
109109
return nil
110110
}
111+
112+
func updateSubnetPrefix(cnsNetworkConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) {
113+
}
114+
115+
func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) {
116+
return nwCfg.Name, nil
117+
}

‎cni/network/network_windows.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66
"net"
7+
"strconv"
78
"strings"
89

910
"github.com/Azure/azure-container-networking/cni"
@@ -70,14 +71,59 @@ func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Resu
7071
}
7172

7273
func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) {
74+
if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
75+
log.Printf("Setting Network Options")
76+
vlanMap := make(map[string]interface{})
77+
vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID)
78+
nwInfo.Options[dockerNetworkOption] = vlanMap
79+
}
7380
}
7481

7582
func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) {
83+
if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
84+
log.Printf("Setting Endpoint Options")
85+
var cnetAddressMap []string
86+
for _, ipSubnet := range cnsNwConfig.CnetAddressSpace {
87+
cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength)))
88+
}
89+
epInfo.Data[network.CnetAddressSpace] = cnetAddressMap
90+
}
7691
}
7792

7893
func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) {
7994
}
8095

96+
func updateSubnetPrefix(cnsNwConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) {
97+
if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
98+
ipconfig := cnsNwConfig.IPConfiguration
99+
ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress)
100+
101+
if ipAddr.To4() != nil {
102+
*subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)}
103+
} else {
104+
*subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)}
105+
}
106+
107+
subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)
108+
log.Printf("Updated subnetPrefix: %s", subnetPrefix.String())
109+
}
110+
}
111+
112+
func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) {
113+
if nwCfg.MultiTenancy {
114+
_, cnsNetworkConfig, _, err := getContainerNetworkConfiguration(nwCfg, "", podName, podNs, ifName)
115+
if err != nil {
116+
log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", podName, podNs, err)
117+
return "", err
118+
}
119+
120+
networkName := fmt.Sprintf("%s-vlanid%v", nwCfg.Name, cnsNetworkConfig.MultiTenancyInfo.ID)
121+
return networkName, nil
122+
}
123+
124+
return nwCfg.Name, nil
125+
}
126+
81127
func setupInfraVnetRoutingForMultitenancy(
82128
nwCfg *cni.NetworkConfig,
83129
azIpamResult *cniTypesCurr.Result,

‎network/endpoint_windows.go‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s
4444

4545
// newEndpointImpl creates a new endpoint in the network.
4646
func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
47+
var vlanid int
48+
49+
if epInfo.Data != nil {
50+
if _, ok := epInfo.Data[VlanIDKey]; ok {
51+
vlanid = epInfo.Data[VlanIDKey].(int)
52+
}
53+
}
54+
4755
// Get Infrastructure containerID. Handle ADD calls for workload container.
4856
var err error
4957
infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName)
@@ -53,7 +61,35 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
5361
VirtualNetwork: nw.HnsId,
5462
DNSSuffix: epInfo.DNS.Suffix,
5563
DNSServerList: strings.Join(epInfo.DNS.Servers, ","),
56-
Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies),
64+
}
65+
66+
// Set outbound NAT policy
67+
outBoundNatPolicy := hcsshim.OutboundNatPolicy{}
68+
outBoundNatPolicy.Policy.Type = hcsshim.OutboundNat
69+
70+
exceptionList, err := policy.GetOutBoundNatExceptionList(epInfo.Policies)
71+
if err != nil {
72+
log.Printf("[net] Failed to parse outbound NAT policy %v", err)
73+
return nil, err
74+
}
75+
76+
if exceptionList != nil {
77+
for _, ipAddress := range exceptionList {
78+
outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress)
79+
}
80+
}
81+
82+
if epInfo.Data[CnetAddressSpace] != nil {
83+
if cnetAddressSpace := epInfo.Data[CnetAddressSpace].([]string); cnetAddressSpace != nil {
84+
for _, ipAddress := range cnetAddressSpace {
85+
outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress)
86+
}
87+
}
88+
}
89+
90+
if outBoundNatPolicy.Exceptions != nil {
91+
serializedOutboundNatPolicy, _ := json.Marshal(outBoundNatPolicy)
92+
hnsEndpoint.Policies = append(hnsEndpoint.Policies, serializedOutboundNatPolicy)
5793
}
5894

5995
// HNS currently supports only one IP address per endpoint.
@@ -96,13 +132,15 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
96132

97133
// Create the endpoint object.
98134
ep := &endpoint{
99-
Id: infraEpName,
100-
HnsId: hnsResponse.Id,
101-
SandboxKey: epInfo.ContainerID,
102-
IfName: epInfo.IfName,
103-
IPAddresses: epInfo.IPAddresses,
104-
Gateways: []net.IP{net.ParseIP(hnsResponse.GatewayAddress)},
105-
DNS: epInfo.DNS,
135+
Id: infraEpName,
136+
HnsId: hnsResponse.Id,
137+
SandboxKey: epInfo.ContainerID,
138+
IfName: epInfo.IfName,
139+
IPAddresses: epInfo.IPAddresses,
140+
Gateways: []net.IP{net.ParseIP(hnsResponse.GatewayAddress)},
141+
DNS: epInfo.DNS,
142+
VlanID: vlanid,
143+
EnableSnatOnHost: epInfo.EnableSnatOnHost,
106144
}
107145

108146
for _, route := range epInfo.Routes {

‎network/manager.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ import (
1515

1616
const (
1717
// Network store key.
18-
storeKey = "Network"
19-
VlanIDKey = "VlanID"
18+
storeKey = "Network"
19+
VlanIDKey = "VlanID"
20+
genericData = "com.docker.network.generic"
2021
)
2122

2223
type NetworkClient interface {

‎network/network.go‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package network
55

66
import (
77
"net"
8+
"strings"
89

910
"github.com/Azure/azure-container-networking/log"
1011
"github.com/Azure/azure-container-networking/network/policy"
@@ -46,6 +47,7 @@ type network struct {
4647

4748
// NetworkInfo contains read-only information about a container network.
4849
type NetworkInfo struct {
50+
MasterIfName string
4951
Id string
5052
Mode string
5153
Subnets []SubnetInfo
@@ -121,6 +123,16 @@ func (nm *networkManager) findExternalInterfaceBySubnet(subnet string) *external
121123
return nil
122124
}
123125

126+
// FindExternalInterfaceByName finds an external interface by name.
127+
func (nm *networkManager) findExternalInterfaceByName(ifName string) *externalInterface {
128+
extIf, exists := nm.ExternalInterfaces[ifName]
129+
if exists && extIf != nil {
130+
return extIf
131+
}
132+
133+
return nil
134+
}
135+
124136
// NewNetwork creates a new container network.
125137
func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) {
126138
var nw *network
@@ -138,8 +150,14 @@ func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) {
138150
nwInfo.Mode = opModeDefault
139151
}
140152

141-
// Find the external interface for this subnet.
142-
extIf := nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String())
153+
// If the master interface name is provided, find the external interface by name
154+
// else use subnet to to find the interface
155+
var extIf *externalInterface
156+
if len(strings.TrimSpace(nwInfo.MasterIfName)) > 0 {
157+
extIf = nm.findExternalInterfaceByName(nwInfo.MasterIfName)
158+
} else {
159+
extIf = nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String())
160+
}
143161
if extIf == nil {
144162
err = errSubnetNotFound
145163
return nil, err

‎network/network_linux.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ const (
2121
// Virtual MAC address used by Azure VNET.
2222
virtualMacAddress = "12:34:56:78:9a:bc"
2323

24-
genericData = "com.docker.network.generic"
25-
2624
SnatBridgeIPKey = "snatBridgeIP"
2725

2826
LocalIPKey = "localIP"

0 commit comments

Comments
 (0)