Skip to content

Commit 9878cab

Browse files
author
Jing Zhang
committed
Get IP addresses of neutron subports
Vlan-aware VMs are commonly used in telecom. Those VMs are plugged into flat networks and use neutron trunk. https://docs.openstack.org/neutron/latest/admin/config-trunking.html Currenlty, nodeAddresses() only returns neutron ports directly attached to VM. For vlan-aware VMs, IP addresses are assigned on neutron subports. Subports are attached to the trunk; they are not attached to the VM directly. This pull request changes nodeAddresses() to return IP addresses of neutron subports when they exist. Without this change, Kubernetes is unable to IP addresses of vlan-aware VMs. This change is transparent to VMs not using neturon trunk. Signed-off-by: Jing Zhang <[email protected]>
1 parent 8a0e73f commit 9878cab

File tree

4 files changed

+98
-39
lines changed

4 files changed

+98
-39
lines changed

‎pkg/openstack/instances.go‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ import (
2929
"github.com/gophercloud/gophercloud"
3030
"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
3131
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
32-
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
32+
"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
33+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3334
"github.com/gophercloud/gophercloud/pagination"
3435
"github.com/mitchellh/mapstructure"
3536
v1 "k8s.io/api/core/v1"
@@ -43,7 +44,6 @@ import (
4344
"k8s.io/cloud-provider-openstack/pkg/util"
4445
"k8s.io/cloud-provider-openstack/pkg/util/errors"
4546
"k8s.io/cloud-provider-openstack/pkg/util/metadata"
46-
"k8s.io/cloud-provider-openstack/pkg/util/openstack"
4747
)
4848

4949
// Instances encapsulates an implementation of Instances for OpenStack.
@@ -240,7 +240,7 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st
240240
return []v1.NodeAddress{}, err
241241
}
242242

243-
addresses, err := nodeAddresses(server, ports, i.networkingOpts)
243+
addresses, err := nodeAddresses(server, ports, i.network, i.networkingOpts)
244244
if err != nil {
245245
return []v1.NodeAddress{}, err
246246
}
@@ -345,7 +345,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
345345
if err != nil {
346346
return nil, err
347347
}
348-
addresses, err := nodeAddresses(srv, ports, i.networkingOpts)
348+
addresses, err := nodeAddresses(srv, ports, i.network, i.networkingOpts)
349349
if err != nil {
350350
return nil, err
351351
}
@@ -573,7 +573,7 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S
573573
// * access IPs
574574
// * metadata hostname
575575
// * server object Addresses (floating type)
576-
func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
576+
func nodeAddresses(srv *servers.Server, ports []PortWithTrunkDetails, client *gophercloud.ServiceClient, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
577577
addrs := []v1.NodeAddress{}
578578

579579
// parse private IP addresses first in an ordered manner
@@ -633,6 +633,39 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
633633
return nil, err
634634
}
635635

636+
// Add the addresses assigned on subports via trunk
637+
// This exposes the vlan networks to which subports are attached
638+
for _, port := range ports {
639+
for _, subport := range port.TrunkDetails.SubPorts {
640+
p, err := neutronports.Get(client, subport.PortID).Extract()
641+
if err != nil {
642+
klog.Errorf("Failed to get subport %s details: %v", subport.PortID, err)
643+
continue
644+
}
645+
n, err := networks.Get(client, p.NetworkID).Extract()
646+
if err != nil {
647+
klog.Errorf("Failed to get subport %s network details: %v", subport.PortID, err)
648+
continue
649+
}
650+
for _, fixedIP := range p.FixedIPs {
651+
klog.V(5).Infof("Node '%s' is found subport '%s' address '%s/%s'", srv.Name, p.Name, n.Name, fixedIP.IPAddress)
652+
isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil
653+
if !(isIPv6 && networkingOpts.IPv6SupportDisabled) {
654+
addr := Address{IPType: "fixed", Addr: fixedIP.IPAddress}
655+
subportAddresses := map[string][]Address{n.Name: []Address{addr}}
656+
srvAddresses, ok := addresses[n.Name]
657+
if !ok {
658+
addresses[n.Name] = subportAddresses[n.Name]
659+
} else {
660+
// this is to take care the corner case
661+
// where the same network is attached to the node both directly and via trunk
662+
addresses[n.Name] = append(srvAddresses, subportAddresses[n.Name]...)
663+
}
664+
}
665+
}
666+
}
667+
}
668+
636669
networks := make([]string, 0, len(addresses))
637670
for k := range addresses {
638671
networks = append(networks, k)
@@ -683,6 +716,7 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
683716
sortNodeAddresses(addrs, networkingOpts.AddressSortOrder)
684717
}
685718

719+
klog.V(5).Infof("Node '%s' returns addresses '%v'", srv.Name, addrs)
686720
return addrs, nil
687721
}
688722

@@ -697,14 +731,25 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName,
697731
return nil, err
698732
}
699733

700-
return nodeAddresses(&srv.Server, ports, networkingOpts)
734+
return nodeAddresses(&srv.Server, ports, client, networkingOpts)
701735
}
702736

703737
// getAttachedPorts returns a list of ports attached to a server.
704-
func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]ports.Port, error) {
705-
listOpts := ports.ListOpts{
738+
func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]PortWithTrunkDetails, error) {
739+
listOpts := neutronports.ListOpts{
706740
DeviceID: serverID,
707741
}
708742

709-
return openstack.GetPorts(client, listOpts)
743+
var ports []PortWithTrunkDetails
744+
745+
allPages, err := neutronports.List(client, listOpts).AllPages()
746+
if err != nil {
747+
return ports, err
748+
}
749+
err = neutronports.ExtractPortsInto(allPages, &ports)
750+
if err != nil {
751+
return ports, err
752+
}
753+
754+
return ports, nil
710755
}

‎pkg/openstack/instancesv2.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*clo
128128
return nil, err
129129
}
130130

131-
addresses, err := nodeAddresses(&server.Server, ports, i.networkingOpts)
131+
addresses, err := nodeAddresses(&server.Server, ports, i.network, i.networkingOpts)
132132
if err != nil {
133133
return nil, err
134134
}

‎pkg/openstack/openstack.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
"github.com/gophercloud/gophercloud"
2828
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones"
2929
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
30+
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunk_details"
31+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3032
"github.com/spf13/pflag"
3133
gcfg "gopkg.in/gcfg.v1"
3234
"k8s.io/apimachinery/pkg/types"
@@ -67,6 +69,11 @@ func AddExtraFlags(fs *pflag.FlagSet) {
6769
fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.")
6870
}
6971

72+
type PortWithTrunkDetails struct {
73+
neutronports.Port
74+
trunk_details.TrunkDetailsExt
75+
}
76+
7077
// LoadBalancer is used for creating and maintaining load balancers
7178
type LoadBalancer struct {
7279
secret *gophercloud.ServiceClient

‎pkg/openstack/openstack_test.go‎

Lines changed: 36 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828

2929
"github.com/gophercloud/gophercloud"
3030
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
31-
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
31+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3232
"github.com/spf13/pflag"
3333
v1 "k8s.io/api/core/v1"
3434
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -360,10 +360,10 @@ func TestNodeAddresses(t *testing.T) {
360360
PublicNetworkName: []string{"public"},
361361
}
362362

363-
ports := []ports.Port{
364-
{
363+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
364+
Port: neutronports.Port{
365365
Status: "ACTIVE",
366-
FixedIPs: []ports.IP{
366+
FixedIPs: []neutronports.IP{
367367
{
368368
IPAddress: "10.0.0.32",
369369
},
@@ -372,9 +372,10 @@ func TestNodeAddresses(t *testing.T) {
372372
},
373373
},
374374
},
375+
},
375376
}
376377

377-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
378+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
378379
if err != nil {
379380
t.Fatalf("nodeAddresses returned error: %v", err)
380381
}
@@ -439,10 +440,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
439440
PublicNetworkName: []string{"pub-net"},
440441
}
441442

442-
ports := []ports.Port{
443-
{
443+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
444+
Port: neutronports.Port{
444445
Status: "ACTIVE",
445-
FixedIPs: []ports.IP{
446+
FixedIPs: []neutronports.IP{
446447
{
447448
IPAddress: "10.0.0.32",
448449
},
@@ -451,9 +452,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
451452
},
452453
},
453454
},
455+
},
454456
}
455457

456-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
458+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
457459
if err != nil {
458460
t.Fatalf("nodeAddresses returned error: %v", err)
459461
}
@@ -512,10 +514,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
512514
PublicNetworkName: []string{"pub-net"},
513515
}
514516

515-
ports := []ports.Port{
516-
{
517+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
518+
Port: neutronports.Port{
517519
Status: "ACTIVE",
518-
FixedIPs: []ports.IP{
520+
FixedIPs: []neutronports.IP{
519521
{
520522
IPAddress: "10.0.0.32",
521523
},
@@ -528,9 +530,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
528530
},
529531
},
530532
},
533+
},
531534
}
532535

533-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
536+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
534537
if err != nil {
535538
t.Fatalf("nodeAddresses returned error: %v", err)
536539
}
@@ -600,10 +603,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
600603
InternalNetworkName: []string{"private", "also-private"},
601604
}
602605

603-
ports := []ports.Port{
604-
{
606+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
607+
Port: neutronports.Port{
605608
Status: "ACTIVE",
606-
FixedIPs: []ports.IP{
609+
FixedIPs: []neutronports.IP{
607610
{
608611
IPAddress: "10.0.0.32",
609612
},
@@ -612,9 +615,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
612615
},
613616
},
614617
},
618+
},
615619
}
616620

617-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
621+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
618622
if err != nil {
619623
t.Fatalf("nodeAddresses returned error: %v", err)
620624
}
@@ -684,10 +688,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
684688
InternalNetworkName: []string{"also-private"},
685689
}
686690

687-
ports := []ports.Port{
688-
{
691+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
692+
Port: neutronports.Port{
689693
Status: "ACTIVE",
690-
FixedIPs: []ports.IP{
694+
FixedIPs: []neutronports.IP{
691695
{
692696
IPAddress: "10.0.0.32",
693697
},
@@ -696,9 +700,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
696700
},
697701
},
698702
},
703+
},
699704
}
700705

701-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
706+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
702707
if err != nil {
703708
t.Fatalf("nodeAddresses returned error: %v", err)
704709
}
@@ -760,10 +765,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
760765
IPv6SupportDisabled: true,
761766
}
762767

763-
ports := []ports.Port{
764-
{
768+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
769+
Port: neutronports.Port{
765770
Status: "ACTIVE",
766-
FixedIPs: []ports.IP{
771+
FixedIPs: []neutronports.IP{
767772
{
768773
IPAddress: "10.0.0.32",
769774
},
@@ -772,9 +777,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
772777
},
773778
},
774779
},
780+
},
775781
}
776782

777-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
783+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
778784
if err != nil {
779785
t.Fatalf("nodeAddresses returned error: %v", err)
780786
}
@@ -841,10 +847,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
841847
AddressSortOrder: "10.0.0.0/8, 50.56.176.0/24, 2001:4800::/32",
842848
}
843849

844-
ports := []ports.Port{
845-
{
850+
ports := []PortWithTrunkDetails{PortWithTrunkDetails{
851+
Port: neutronports.Port{
846852
Status: "ACTIVE",
847-
FixedIPs: []ports.IP{
853+
FixedIPs: []neutronports.IP{
848854
{
849855
IPAddress: "10.0.0.32",
850856
},
@@ -853,9 +859,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
853859
},
854860
},
855861
},
862+
},
856863
}
857864

858-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
865+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
859866
if err != nil {
860867
t.Fatalf("nodeAddresses returned error: %v", err)
861868
}

0 commit comments

Comments
 (0)