From 3e01293afe12056f8a05fc2f66101b8082070b99 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Tue, 28 Mar 2023 12:14:54 -0500 Subject: [PATCH] Get IP addresses of neutron subports Vlan-aware VMs are commonly used in telecom. Those VMs are plugged into flat networks and use neutron trunk. https://docs.openstack.org/neutron/latest/admin/config-trunking.html Currenlty, nodeAddresses() only returns neutron ports directly attached to VM. For vlan-aware VMs, IP addresses are assigned on neutron subports. Subports are attached to the trunk; they are not attached to the VM directly. This pull request changes nodeAddresses() to return IP addresses of neutron subports when they exist. Without this change, Kubernetes is unable to IP addresses of vlan-aware VMs. This change is transparent to VMs not using neturon trunk. Signed-off-by: Jing Zhang --- pkg/openstack/instances.go | 63 +++++++++++++++++++++++++++----- pkg/openstack/instancesv2.go | 2 +- pkg/openstack/openstack.go | 7 ++++ pkg/openstack/openstack_test.go | 65 ++++++++++++++++++--------------- 4 files changed, 98 insertions(+), 39 deletions(-) diff --git a/pkg/openstack/instances.go b/pkg/openstack/instances.go index 7cb4389198..60c7745411 100644 --- a/pkg/openstack/instances.go +++ b/pkg/openstack/instances.go @@ -29,7 +29,8 @@ import ( "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" + neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/gophercloud/gophercloud/pagination" "github.com/mitchellh/mapstructure" v1 "k8s.io/api/core/v1" @@ -43,7 +44,6 @@ import ( "k8s.io/cloud-provider-openstack/pkg/util" "k8s.io/cloud-provider-openstack/pkg/util/errors" "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/openstack" ) // Instances encapsulates an implementation of Instances for OpenStack. @@ -240,7 +240,7 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st return []v1.NodeAddress{}, err } - addresses, err := nodeAddresses(server, ports, i.networkingOpts) + addresses, err := nodeAddresses(server, ports, i.network, i.networkingOpts) if err != nil { return []v1.NodeAddress{}, err } @@ -345,7 +345,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud if err != nil { return nil, err } - addresses, err := nodeAddresses(srv, ports, i.networkingOpts) + addresses, err := nodeAddresses(srv, ports, i.network, i.networkingOpts) if err != nil { return nil, err } @@ -573,7 +573,7 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S // * access IPs // * metadata hostname // * server object Addresses (floating type) -func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) { +func nodeAddresses(srv *servers.Server, ports []PortWithTrunkDetails, client *gophercloud.ServiceClient, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) { addrs := []v1.NodeAddress{} // parse private IP addresses first in an ordered manner @@ -633,6 +633,39 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo return nil, err } + // Add the addresses assigned on subports via trunk + // This exposes the vlan networks to which subports are attached + for _, port := range ports { + for _, subport := range port.TrunkDetails.SubPorts { + p, err := neutronports.Get(client, subport.PortID).Extract() + if err != nil { + klog.Errorf("Failed to get subport %s details: %v", subport.PortID, err) + continue + } + n, err := networks.Get(client, p.NetworkID).Extract() + if err != nil { + klog.Errorf("Failed to get subport %s network details: %v", subport.PortID, err) + continue + } + for _, fixedIP := range p.FixedIPs { + klog.V(5).Infof("Node '%s' is found subport '%s' address '%s/%s'", srv.Name, p.Name, n.Name, fixedIP.IPAddress) + isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil + if !(isIPv6 && networkingOpts.IPv6SupportDisabled) { + addr := Address{IPType: "fixed", Addr: fixedIP.IPAddress} + subportAddresses := map[string][]Address{n.Name: {addr}} + srvAddresses, ok := addresses[n.Name] + if !ok { + addresses[n.Name] = subportAddresses[n.Name] + } else { + // this is to take care the corner case + // where the same network is attached to the node both directly and via trunk + addresses[n.Name] = append(srvAddresses, subportAddresses[n.Name]...) + } + } + } + } + } + networks := make([]string, 0, len(addresses)) for k := range addresses { networks = append(networks, k) @@ -683,6 +716,7 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo sortNodeAddresses(addrs, networkingOpts.AddressSortOrder) } + klog.V(5).Infof("Node '%s' returns addresses '%v'", srv.Name, addrs) return addrs, nil } @@ -697,14 +731,25 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName, return nil, err } - return nodeAddresses(&srv.Server, ports, networkingOpts) + return nodeAddresses(&srv.Server, ports, client, networkingOpts) } // getAttachedPorts returns a list of ports attached to a server. -func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]ports.Port, error) { - listOpts := ports.ListOpts{ +func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]PortWithTrunkDetails, error) { + listOpts := neutronports.ListOpts{ DeviceID: serverID, } - return openstack.GetPorts(client, listOpts) + var ports []PortWithTrunkDetails + + allPages, err := neutronports.List(client, listOpts).AllPages() + if err != nil { + return ports, err + } + err = neutronports.ExtractPortsInto(allPages, &ports) + if err != nil { + return ports, err + } + + return ports, nil } diff --git a/pkg/openstack/instancesv2.go b/pkg/openstack/instancesv2.go index eea86dcc2b..3e6f770f7e 100644 --- a/pkg/openstack/instancesv2.go +++ b/pkg/openstack/instancesv2.go @@ -128,7 +128,7 @@ func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*clo return nil, err } - addresses, err := nodeAddresses(&server.Server, ports, i.networkingOpts) + addresses, err := nodeAddresses(&server.Server, ports, i.network, i.networkingOpts) if err != nil { return nil, err } diff --git a/pkg/openstack/openstack.go b/pkg/openstack/openstack.go index 02bbe4e9f8..dee30bb624 100644 --- a/pkg/openstack/openstack.go +++ b/pkg/openstack/openstack.go @@ -27,6 +27,8 @@ import ( "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunk_details" + neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/spf13/pflag" gcfg "gopkg.in/gcfg.v1" "k8s.io/apimachinery/pkg/types" @@ -67,6 +69,11 @@ func AddExtraFlags(fs *pflag.FlagSet) { fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.") } +type PortWithTrunkDetails struct { + neutronports.Port + trunk_details.TrunkDetailsExt +} + // LoadBalancer is used for creating and maintaining load balancers type LoadBalancer struct { secret *gophercloud.ServiceClient diff --git a/pkg/openstack/openstack_test.go b/pkg/openstack/openstack_test.go index cd31f80c14..57bf9fccbc 100644 --- a/pkg/openstack/openstack_test.go +++ b/pkg/openstack/openstack_test.go @@ -28,7 +28,7 @@ import ( "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" + neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -360,10 +360,10 @@ func TestNodeAddresses(t *testing.T) { PublicNetworkName: []string{"public"}, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -372,9 +372,10 @@ func TestNodeAddresses(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -439,10 +440,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) { PublicNetworkName: []string{"pub-net"}, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -451,9 +452,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -512,10 +514,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) { PublicNetworkName: []string{"pub-net"}, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -528,9 +530,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -600,10 +603,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) { InternalNetworkName: []string{"private", "also-private"}, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -612,9 +615,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -684,10 +688,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) { InternalNetworkName: []string{"also-private"}, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -696,9 +700,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -760,10 +765,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) { IPv6SupportDisabled: true, } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -772,9 +777,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) } @@ -841,10 +847,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) { AddressSortOrder: "10.0.0.0/8, 50.56.176.0/24, 2001:4800::/32", } - ports := []ports.Port{ - { + ports := []PortWithTrunkDetails{{ + Port: neutronports.Port{ Status: "ACTIVE", - FixedIPs: []ports.IP{ + FixedIPs: []neutronports.IP{ { IPAddress: "10.0.0.32", }, @@ -853,9 +859,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) { }, }, }, + }, } - addrs, err := nodeAddresses(&srv, ports, networkingOpts) + addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts) if err != nil { t.Fatalf("nodeAddresses returned error: %v", err) }