Skip to content

Commit d62cd1a

Browse files
sunzhaohua2sunzhaohua2
committed
check load balancer healthcheck port and path
1 parent 4581866 commit d62cd1a

File tree

4 files changed

+159
-0
lines changed

4 files changed

+159
-0
lines changed

‎test/extended/cloud_controller_manager/ccm.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"strings"
8+
"time"
79

810
"github.com/ghodss/yaml"
911
g "github.com/onsi/ginkgo/v2"
@@ -12,6 +14,8 @@ import (
1214
exutil "github.com/openshift/origin/test/extended/util"
1315
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1416
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
17+
"k8s.io/apimachinery/pkg/util/wait"
18+
e2e "k8s.io/kubernetes/test/e2e/framework"
1519
)
1620

1721
const cloudControllerNamespace = "openshift-cloud-controller-manager"
@@ -88,6 +92,38 @@ var _ = g.Describe("[sig-cloud-provider][Feature:OpenShiftCloudControllerManager
8892
o.HaveField("Contents", o.ContainSubstring("cloud-provider=external")),
8993
)))
9094
})
95+
96+
g.It("Cluster scoped load balancer healthcheck port and path should be 10256/healthz", func() {
97+
exutil.SkipIfNotPlatform(oc, "AWS")
98+
if strings.HasPrefix(exutil.GetClusterRegion(oc), "us-iso") {
99+
g.Skip("Skipped: There is no public subnet on AWS C2S/SC2S disconnected clusters!")
100+
}
101+
102+
g.By("Create a cluster scope load balancer")
103+
svcName := "test-lb"
104+
defer oc.WithoutNamespace().AsAdmin().Run("delete").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--ignore-not-found").Execute()
105+
out, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--tcp=80:8080").Output()
106+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create lb service")
107+
o.Expect(out).To(o.ContainSubstring("service/" + svcName + " created"))
108+
109+
g.By("Check External-IP assigned")
110+
svcExternalIP := getLoadBalancerExternalIP(oc, oc.Namespace(), svcName)
111+
e2e.Logf("External IP assigned: %s", svcExternalIP)
112+
o.Expect(svcExternalIP).NotTo(o.BeEmpty(), "externalIP should not be empty")
113+
lbName := strings.Split(svcExternalIP, "-")[0]
114+
115+
g.By("Check healthcheck port and path should be 10256/healthz")
116+
healthCheckPort := "10256"
117+
healthCheckPath := "/healthz"
118+
exutil.GetAwsCredentialFromCluster(oc)
119+
region := exutil.GetClusterRegion(oc)
120+
sess := exutil.InitAwsSession(region)
121+
elbClient := exutil.NewELBClient(sess)
122+
healthCheck, err := elbClient.GetLBHealthCheckPortPath(lbName)
123+
o.Expect(err).NotTo(o.HaveOccurred(), "unable to get health check port and path")
124+
e2e.Logf("Health check port and path: %v", healthCheck)
125+
o.Expect(healthCheck).To(o.Equal(fmt.Sprintf("HTTP:%s%s", healthCheckPort, healthCheckPath)))
126+
})
91127
})
92128

93129
// isPlatformExternal returns true when the platform has an in-tree provider,
@@ -103,3 +139,19 @@ func isPlatformExternal(platformType configv1.PlatformType) bool {
103139
return false
104140
}
105141
}
142+
143+
// getLoadBalancerExternalIP get IP address of LB service
144+
func getLoadBalancerExternalIP(oc *exutil.CLI, namespace string, svcName string) string {
145+
var svcExternalIP string
146+
var cmdErr error
147+
checkErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) {
148+
svcExternalIP, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}").Output()
149+
if svcExternalIP == "" || cmdErr != nil {
150+
e2e.Logf("Waiting for lb service IP assignment. Trying again...")
151+
return false, nil
152+
}
153+
return true, nil
154+
})
155+
o.Expect(checkErr).NotTo(o.HaveOccurred())
156+
return svcExternalIP
157+
}

‎test/extended/util/annotate/generated/zz_generated.annotations.go‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/extended/util/aws_client.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package util
2+
3+
import (
4+
"encoding/base64"
5+
"os"
6+
7+
"github.com/aws/aws-sdk-go/aws"
8+
"github.com/aws/aws-sdk-go/aws/session"
9+
"github.com/aws/aws-sdk-go/service/elb"
10+
g "github.com/onsi/ginkgo/v2"
11+
o "github.com/onsi/gomega"
12+
"github.com/tidwall/gjson"
13+
14+
e2e "k8s.io/kubernetes/test/e2e/framework"
15+
)
16+
17+
// GetAwsCredentialFromCluster get aws credential from cluster
18+
func GetAwsCredentialFromCluster(oc *CLI) {
19+
credential, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output()
20+
// Skip for sts and c2s clusters.
21+
if err != nil {
22+
g.Skip("Did not get credential to access aws, skip the testing.")
23+
}
24+
o.Expect(err).NotTo(o.HaveOccurred())
25+
accessKeyIDBase64, secureKeyBase64 := gjson.Get(credential, `data.aws_access_key_id`).String(), gjson.Get(credential, `data.aws_secret_access_key`).String()
26+
accessKeyID, err1 := base64.StdEncoding.DecodeString(accessKeyIDBase64)
27+
o.Expect(err1).NotTo(o.HaveOccurred())
28+
secureKey, err2 := base64.StdEncoding.DecodeString(secureKeyBase64)
29+
o.Expect(err2).NotTo(o.HaveOccurred())
30+
clusterRegion, err3 := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output()
31+
o.Expect(err3).NotTo(o.HaveOccurred())
32+
os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID))
33+
os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey))
34+
os.Setenv("AWS_REGION", clusterRegion)
35+
}
36+
37+
// InitAwsSession init session
38+
func InitAwsSession(region string) *session.Session {
39+
sess := session.Must(session.NewSessionWithOptions(session.Options{
40+
Config: aws.Config{
41+
Region: aws.String(region),
42+
},
43+
}))
44+
45+
return sess
46+
}
47+
48+
type ELBClient struct {
49+
svc *elb.ELB
50+
}
51+
52+
// NewELBClient creates an ECRClient
53+
func NewELBClient(sess *session.Session) *ELBClient {
54+
return &ELBClient{
55+
svc: elb.New(sess),
56+
}
57+
}
58+
59+
// GetLBHealthCheckPortPath get load balance health check port and path
60+
func (elbClient *ELBClient) GetLBHealthCheckPortPath(lbName string) (string, error) {
61+
input := &elb.DescribeLoadBalancersInput{
62+
LoadBalancerNames: []*string{
63+
aws.String(lbName),
64+
},
65+
}
66+
67+
result, err := elbClient.svc.DescribeLoadBalancers(input)
68+
if err != nil {
69+
e2e.Logf("Failed to describe load balancer: %v", err)
70+
return "", err
71+
}
72+
73+
if len(result.LoadBalancerDescriptions) == 0 {
74+
e2e.Logf("Failed to get load balancers: %v", err)
75+
}
76+
77+
healthCheck := result.LoadBalancerDescriptions[0].HealthCheck
78+
if healthCheck == nil {
79+
e2e.Logf("Failed to get health check: %v", err)
80+
}
81+
return *healthCheck.Target, nil
82+
}

‎test/extended/util/framework.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2403,3 +2403,26 @@ func IsCapabilityEnabled(oc *CLI, cap configv1.ClusterVersionCapability) (bool,
24032403
}
24042404
return false, nil
24052405
}
2406+
2407+
// SkipIfNotPlatform skip the test if supported platforms are not matched
2408+
func SkipIfNotPlatform(oc *CLI, platforms ...configv1.PlatformType) {
2409+
var match bool
2410+
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
2411+
o.Expect(err).NotTo(o.HaveOccurred())
2412+
for _, platform := range platforms {
2413+
if infra.Status.PlatformStatus.Type == platform {
2414+
match = true
2415+
break
2416+
}
2417+
}
2418+
if !match {
2419+
g.Skip("Skip this test scenario because it is not supported on the " + string(infra.Status.PlatformStatus.Type) + " platform")
2420+
}
2421+
}
2422+
2423+
// GetClusterRegion get the cluster's region
2424+
func GetClusterRegion(oc *CLI) string {
2425+
region, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", `-ojsonpath={.items[].metadata.labels.topology\.kubernetes\.io/region}`).Output()
2426+
o.Expect(err).NotTo(o.HaveOccurred())
2427+
return region
2428+
}

0 commit comments

Comments
 (0)