Merge pull request #241 from chengxiangdong/feat_nic

feat: Add support for querying the primary network

Author: k8s-ci-robot

docs/huawei-cloud-controller-manager-configuration.md

@@ -233,4 +233,9 @@ The following arguments are supported:
   which may ultimately cause CCM to malfunction. 
   For example, `business-name: order-pass`, kubernetes loadbalancer service namespace/name: `default/order-service`,
   then the name of the ELB instance is: `k8s_service_order-pass_default_order-service`.
+
+  > Note:
   > Changing this will create new ELB instances, the old ELB instances will not be deleted and no longer maintained.
+
+* `primary-nic` Optional. If you want to use the node's primary network card as the back-end service of ELB,
+  please configure `force`, otherwise use HostIP of pod.

pkg/cloudprovider/huaweicloud/dedicatedloadbalancer.go

@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/status"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
@@ -538,7 +539,7 @@ func (d *DedicatedLoadBalancer) addOrRemoveMembers(loadbalancer *elbmodel.LoadBa
 				pod.Namespace, pod.Spec.NodeName)
 		}
 
-		address, portNum, err := getMemberIP(service, node, pod, svcPort)
+		address, portNum, err := d.getMemberIP(service, node, pod, svcPort)
 		if err != nil {
 			if common.IsNotFound(err) {
 				// Node failure, do not create member
@@ -581,7 +582,7 @@ func (d *DedicatedLoadBalancer) addOrRemoveMembers(loadbalancer *elbmodel.LoadBa
 
 func (d *DedicatedLoadBalancer) addMember(service *v1.Service, loadbalancer *elbmodel.LoadBalancer, pool *elbmodel.Pool, pod v1.Pod, svcPort v1.ServicePort, node *v1.Node) error {
 	klog.Infof("Add a member(%s) to pool %s", node.Name, pool.Id)
-	address, port, err := getMemberIP(service, node, pod, svcPort)
+	address, port, err := d.getMemberIP(service, node, pod, svcPort)
 	if err != nil {
 		return err
 	}
@@ -613,6 +614,48 @@ func (d *DedicatedLoadBalancer) addMember(service *v1.Service, loadbalancer *elb
 	return nil
 }
 
+func (d *DedicatedLoadBalancer) getMemberIP(service *v1.Service, node *v1.Node, pod v1.Pod, svcPort v1.ServicePort) (string, int32, error) {
+	if service.Spec.AllocateLoadBalancerNodePorts != nil && *service.Spec.AllocateLoadBalancerNodePorts {
+		klog.Infof("add member using the Node's IP and port, service: %s/%s, port: %s ", service.Namespace, service.Name, svcPort.Name)
+
+		address := ""
+		if pod.Status.HostIP != "" {
+			address = pod.Status.HostIP
+		} else {
+			addr, err := getNodeAddress(node)
+			if err != nil {
+				return "", 0, err
+			}
+			address = addr
+		}
+
+		address, err := d.getPrimaryIP(address)
+		if err != nil {
+			return "", 0, err
+		}
+		return address, svcPort.NodePort, nil
+	}
+
+	if service.Spec.AllocateLoadBalancerNodePorts != nil && !*service.Spec.AllocateLoadBalancerNodePorts {
+		klog.Infof("add member using the Pod's IP and port, service: %s/%s, port: %s ", service.Namespace, service.Name, svcPort.Name)
+		// get IP and port from Pod
+		if svcPort.TargetPort.Type == intstr.Int {
+			klog.V(6).Infof("targetPort is a number, service: %s/%s, port: %s ", service.Namespace, service.Name, svcPort.Name)
+			return pod.Status.PodIP, svcPort.TargetPort.IntVal, nil
+		}
+
+		klog.V(6).Infof("targetPort is a name, service: %s/%s, port: %s ", service.Namespace, service.Name, svcPort.Name)
+		for _, c := range pod.Spec.Containers {
+			for _, p := range c.Ports {
+				if p.Name == svcPort.TargetPort.StrVal && string(p.Protocol) == string(svcPort.Protocol) {
+					return pod.Status.PodIP, p.ContainerPort, nil
+				}
+			}
+		}
+	}
+	return "", 0, fmt.Errorf("not found member IP and port")
+}
+
 func (d *DedicatedLoadBalancer) deleteMember(elbID string, poolID string, member elbmodel.Member) error {
 	klog.V(4).Infof("Deleting exists member %s for pool %s address %s", member.Id, poolID, member.Address)
 	err := d.dedicatedELBClient.DeleteMember(poolID, member.Id)

pkg/cloudprovider/huaweicloud/huaweicloud.go

@@ -350,6 +350,26 @@ func (b Basic) isSupportedSvc(svs *v1.Service) bool {
 	return true
 }
 
+func (b Basic) getPrimaryIP(ip string) (string, error) {
+	if b.loadbalancerOpts.PrimaryNic != "force" {
+		return ip, nil
+	}
+
+	instance, err := b.ecsClient.GetByNodeIPNew(ip)
+	if err != nil {
+		return "", err
+	}
+	for _, arr := range instance.Addresses {
+		for _, v := range arr {
+			if v.Primary {
+				klog.Infof("obtain the ECS details through the private IP: %s, and find the primary network card IP: %s", ip, v.Addr)
+				return v.Addr, nil
+			}
+		}
+	}
+	return "", status.Errorf(codes.NotFound, "not found ECS primary network by private ip: %s", ip)
+}
+
 type CloudProvider struct {
 	Basic
 	providers map[LoadBalanceVersion]cloudprovider.LoadBalancer

pkg/cloudprovider/huaweicloud/model/ecs_meta.go

@@ -0,0 +1,67 @@
+package model
+
+import (
+	"net/http"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/def"
+)
+
+func GenReqDefForListServersDetails() *def.HttpRequestDef {
+	reqDefBuilder := def.NewHttpRequestDefBuilder().
+		WithMethod(http.MethodGet).
+		WithPath("/v1/{project_id}/cloudservers/detail").
+		WithResponse(new(ListServersDetailsResponse)).
+		WithContentType("application/json")
+
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("EnterpriseProjectId").
+		WithJsonTag("enterprise_project_id").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Flavor").
+		WithJsonTag("flavor").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Ip").
+		WithJsonTag("ip").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Limit").
+		WithJsonTag("limit").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Name").
+		WithJsonTag("name").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("NotTags").
+		WithJsonTag("not-tags").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Offset").
+		WithJsonTag("offset").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("ReservationId").
+		WithJsonTag("reservation_id").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Status").
+		WithJsonTag("status").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("Tags").
+		WithJsonTag("tags").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("IpEq").
+		WithJsonTag("ip_eq").
+		WithLocationType(def.Query))
+	reqDefBuilder.WithRequestField(def.NewFieldDef().
+		WithName("ServerId").
+		WithJsonTag("server_id").
+		WithLocationType(def.Query))
+
+	requestDef := reqDefBuilder.Build()
+	return requestDef
+}

pkg/cloudprovider/huaweicloud/model/model_list_servers_details_response.go

@@ -0,0 +1,28 @@
+// nolint: golint
+package model
+
+import (
+	"strings"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/utils"
+)
+
+// Response Object
+type ListServersDetailsResponse struct {
+
+	// 弹性云服务器的列表总数。
+	Count *int32 `json:"count,omitempty"`
+
+	// 弹性云服务器详情列表，具体参照-查询云服务器详情接口。查询级别不同，返回的详情不同。
+	Servers        *[]ServerDetail `json:"servers,omitempty"`
+	HttpStatusCode int             `json:"-"`
+}
+
+func (o ListServersDetailsResponse) String() string {
+	data, err := utils.Marshal(o)
+	if err != nil {
+		return "ListServersDetailsResponse struct{}"
+	}
+
+	return strings.Join([]string{"ListServersDetailsResponse", string(data)}, " ")
+}

pkg/cloudprovider/huaweicloud/model/model_server_address.go

@@ -0,0 +1,84 @@
+// nolint: golint
+package model
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/utils"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/converter"
+)
+
+// 弹性云服务器的网络属性。
+type ServerAddress struct {
+
+	// IP地址版本。  - “4”：代表IPv4。 - “6”：代表IPv6。
+	Version string `json:"version"`
+
+	// IP地址。
+	Addr string `json:"addr"`
+
+	// 是否主网卡
+	Primary bool `json:"primary"`
+
+	// IP地址类型。  - fixed：代表私有IP地址。 - floating：代表浮动IP地址。
+	OSEXTIPStype *ServerAddressOSEXTIPStype `json:"OS-EXT-IPS:type,omitempty"`
+
+	// MAC地址。
+	OSEXTIPSMACmacAddr *string `json:"OS-EXT-IPS-MAC:mac_addr,omitempty"`
+
+	// IP地址对应的端口ID。
+	OSEXTIPSportId *string `json:"OS-EXT-IPS:port_id,omitempty"`
+}
+
+func (o ServerAddress) String() string {
+	data, err := utils.Marshal(o)
+	if err != nil {
+		return "ServerAddress struct{}"
+	}
+
+	return strings.Join([]string{"ServerAddress", string(data)}, " ")
+}
+
+type ServerAddressOSEXTIPStype struct {
+	value string
+}
+
+type ServerAddressOSEXTIPStypeEnum struct {
+	FIXED    ServerAddressOSEXTIPStype
+	FLOATING ServerAddressOSEXTIPStype
+}
+
+func GetServerAddressOSEXTIPStypeEnum() ServerAddressOSEXTIPStypeEnum {
+	return ServerAddressOSEXTIPStypeEnum{
+		FIXED: ServerAddressOSEXTIPStype{
+			value: "fixed",
+		},
+		FLOATING: ServerAddressOSEXTIPStype{
+			value: "floating",
+		},
+	}
+}
+
+func (c ServerAddressOSEXTIPStype) Value() string {
+	return c.value
+}
+
+func (c ServerAddressOSEXTIPStype) MarshalJSON() ([]byte, error) {
+	return utils.Marshal(c.value)
+}
+
+func (c *ServerAddressOSEXTIPStype) UnmarshalJSON(b []byte) error {
+	myConverter := converter.StringConverterFactory("string")
+	if myConverter != nil {
+		val, err := myConverter.CovertStringToInterface(strings.Trim(string(b[:]), "\""))
+		if err == nil {
+			c.value = val.(string)
+			return nil
+		}
+		return err
+	} else {
+		return errors.New("convert enum data to string error")
+	}
+}