Skip to content

Commit a1e274c

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #29544 from sunzhaohua2/lb
OCPQE-28543: Check load balancer healthcheck port and path on aws
2 parents 9b0ebd2 + d62cd1a commit a1e274c

File tree

212 files changed

+82425
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

212 files changed

+82425
-0
lines changed

‎go.mod‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ require (
1010
github.com/MakeNowJust/heredoc v1.0.0
1111
github.com/RangelReale/osincli v0.0.0-20160924135400-fababb0555f2
1212
github.com/apparentlymart/go-cidr v1.1.0
13+
github.com/aws/aws-sdk-go v1.44.204
1314
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
1415
github.com/distribution/distribution/v3 v3.0.0-20230530204932-ba46c769b3d1
1516
github.com/fsouza/go-dockerclient v1.12.0
@@ -47,6 +48,7 @@ require (
4748
github.com/spf13/viper v1.8.1
4849
github.com/stretchr/objx v0.5.2
4950
github.com/stretchr/testify v1.9.0
51+
github.com/tidwall/gjson v1.18.0
5052
go.etcd.io/etcd/client/pkg/v3 v3.5.16
5153
go.etcd.io/etcd/client/v3 v3.5.16
5254
golang.org/x/crypto v0.31.0
@@ -184,6 +186,7 @@ require (
184186
github.com/hashicorp/hcl v1.0.0 // indirect
185187
github.com/inconshreveable/mousetrap v1.1.0 // indirect
186188
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
189+
github.com/jmespath/go-jmespath v0.4.0 // indirect
187190
github.com/jonboulle/clockwork v0.4.0 // indirect
188191
github.com/josharian/intern v1.0.0 // indirect
189192
github.com/json-iterator/go v1.1.12 // indirect
@@ -242,6 +245,8 @@ require (
242245
github.com/src-d/gcfg v1.4.0 // indirect
243246
github.com/stoewer/go-strcase v1.3.0 // indirect
244247
github.com/subosito/gotenv v1.2.0 // indirect
248+
github.com/tidwall/match v1.1.1 // indirect
249+
github.com/tidwall/pretty v1.2.0 // indirect
245250
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect
246251
github.com/x448/float16 v0.8.4 // indirect
247252
github.com/xanzy/ssh-agent v0.2.1 // indirect

‎go.sum‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
148148
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
149149
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
150150
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
151+
github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ=
152+
github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
151153
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
152154
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
153155
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -461,6 +463,10 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
461463
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
462464
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
463465
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
466+
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
467+
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
468+
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
469+
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
464470
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
465471
github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
466472
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -754,6 +760,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
754760
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
755761
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
756762
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
763+
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
764+
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
765+
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
766+
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
767+
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
768+
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
757769
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
758770
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
759771
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -944,6 +956,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
944956
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
945957
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
946958
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
959+
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
947960
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
948961
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
949962
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
@@ -1040,6 +1053,7 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
10401053
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
10411054
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
10421055
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
1056+
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
10431057
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
10441058
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
10451059
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
@@ -1054,6 +1068,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10541068
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10551069
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10561070
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
1071+
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
10571072
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
10581073
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
10591074
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

‎test/extended/cloud_controller_manager/ccm.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"strings"
8+
"time"
79

810
"github.com/ghodss/yaml"
911
g "github.com/onsi/ginkgo/v2"
@@ -12,6 +14,8 @@ import (
1214
exutil "github.com/openshift/origin/test/extended/util"
1315
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1416
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
17+
"k8s.io/apimachinery/pkg/util/wait"
18+
e2e "k8s.io/kubernetes/test/e2e/framework"
1519
)
1620

1721
const cloudControllerNamespace = "openshift-cloud-controller-manager"
@@ -88,6 +92,38 @@ var _ = g.Describe("[sig-cloud-provider][Feature:OpenShiftCloudControllerManager
8892
o.HaveField("Contents", o.ContainSubstring("cloud-provider=external")),
8993
)))
9094
})
95+
96+
g.It("Cluster scoped load balancer healthcheck port and path should be 10256/healthz", func() {
97+
exutil.SkipIfNotPlatform(oc, "AWS")
98+
if strings.HasPrefix(exutil.GetClusterRegion(oc), "us-iso") {
99+
g.Skip("Skipped: There is no public subnet on AWS C2S/SC2S disconnected clusters!")
100+
}
101+
102+
g.By("Create a cluster scope load balancer")
103+
svcName := "test-lb"
104+
defer oc.WithoutNamespace().AsAdmin().Run("delete").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--ignore-not-found").Execute()
105+
out, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--tcp=80:8080").Output()
106+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create lb service")
107+
o.Expect(out).To(o.ContainSubstring("service/" + svcName + " created"))
108+
109+
g.By("Check External-IP assigned")
110+
svcExternalIP := getLoadBalancerExternalIP(oc, oc.Namespace(), svcName)
111+
e2e.Logf("External IP assigned: %s", svcExternalIP)
112+
o.Expect(svcExternalIP).NotTo(o.BeEmpty(), "externalIP should not be empty")
113+
lbName := strings.Split(svcExternalIP, "-")[0]
114+
115+
g.By("Check healthcheck port and path should be 10256/healthz")
116+
healthCheckPort := "10256"
117+
healthCheckPath := "/healthz"
118+
exutil.GetAwsCredentialFromCluster(oc)
119+
region := exutil.GetClusterRegion(oc)
120+
sess := exutil.InitAwsSession(region)
121+
elbClient := exutil.NewELBClient(sess)
122+
healthCheck, err := elbClient.GetLBHealthCheckPortPath(lbName)
123+
o.Expect(err).NotTo(o.HaveOccurred(), "unable to get health check port and path")
124+
e2e.Logf("Health check port and path: %v", healthCheck)
125+
o.Expect(healthCheck).To(o.Equal(fmt.Sprintf("HTTP:%s%s", healthCheckPort, healthCheckPath)))
126+
})
91127
})
92128

93129
// isPlatformExternal returns true when the platform has an in-tree provider,
@@ -103,3 +139,19 @@ func isPlatformExternal(platformType configv1.PlatformType) bool {
103139
return false
104140
}
105141
}
142+
143+
// getLoadBalancerExternalIP get IP address of LB service
144+
func getLoadBalancerExternalIP(oc *exutil.CLI, namespace string, svcName string) string {
145+
var svcExternalIP string
146+
var cmdErr error
147+
checkErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) {
148+
svcExternalIP, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}").Output()
149+
if svcExternalIP == "" || cmdErr != nil {
150+
e2e.Logf("Waiting for lb service IP assignment. Trying again...")
151+
return false, nil
152+
}
153+
return true, nil
154+
})
155+
o.Expect(checkErr).NotTo(o.HaveOccurred())
156+
return svcExternalIP
157+
}

‎test/extended/util/annotate/generated/zz_generated.annotations.go‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/extended/util/aws_client.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package util
2+
3+
import (
4+
"encoding/base64"
5+
"os"
6+
7+
"github.com/aws/aws-sdk-go/aws"
8+
"github.com/aws/aws-sdk-go/aws/session"
9+
"github.com/aws/aws-sdk-go/service/elb"
10+
g "github.com/onsi/ginkgo/v2"
11+
o "github.com/onsi/gomega"
12+
"github.com/tidwall/gjson"
13+
14+
e2e "k8s.io/kubernetes/test/e2e/framework"
15+
)
16+
17+
// GetAwsCredentialFromCluster get aws credential from cluster
18+
func GetAwsCredentialFromCluster(oc *CLI) {
19+
credential, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output()
20+
// Skip for sts and c2s clusters.
21+
if err != nil {
22+
g.Skip("Did not get credential to access aws, skip the testing.")
23+
}
24+
o.Expect(err).NotTo(o.HaveOccurred())
25+
accessKeyIDBase64, secureKeyBase64 := gjson.Get(credential, `data.aws_access_key_id`).String(), gjson.Get(credential, `data.aws_secret_access_key`).String()
26+
accessKeyID, err1 := base64.StdEncoding.DecodeString(accessKeyIDBase64)
27+
o.Expect(err1).NotTo(o.HaveOccurred())
28+
secureKey, err2 := base64.StdEncoding.DecodeString(secureKeyBase64)
29+
o.Expect(err2).NotTo(o.HaveOccurred())
30+
clusterRegion, err3 := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output()
31+
o.Expect(err3).NotTo(o.HaveOccurred())
32+
os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID))
33+
os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey))
34+
os.Setenv("AWS_REGION", clusterRegion)
35+
}
36+
37+
// InitAwsSession init session
38+
func InitAwsSession(region string) *session.Session {
39+
sess := session.Must(session.NewSessionWithOptions(session.Options{
40+
Config: aws.Config{
41+
Region: aws.String(region),
42+
},
43+
}))
44+
45+
return sess
46+
}
47+
48+
type ELBClient struct {
49+
svc *elb.ELB
50+
}
51+
52+
// NewELBClient creates an ECRClient
53+
func NewELBClient(sess *session.Session) *ELBClient {
54+
return &ELBClient{
55+
svc: elb.New(sess),
56+
}
57+
}
58+
59+
// GetLBHealthCheckPortPath get load balance health check port and path
60+
func (elbClient *ELBClient) GetLBHealthCheckPortPath(lbName string) (string, error) {
61+
input := &elb.DescribeLoadBalancersInput{
62+
LoadBalancerNames: []*string{
63+
aws.String(lbName),
64+
},
65+
}
66+
67+
result, err := elbClient.svc.DescribeLoadBalancers(input)
68+
if err != nil {
69+
e2e.Logf("Failed to describe load balancer: %v", err)
70+
return "", err
71+
}
72+
73+
if len(result.LoadBalancerDescriptions) == 0 {
74+
e2e.Logf("Failed to get load balancers: %v", err)
75+
}
76+
77+
healthCheck := result.LoadBalancerDescriptions[0].HealthCheck
78+
if healthCheck == nil {
79+
e2e.Logf("Failed to get health check: %v", err)
80+
}
81+
return *healthCheck.Target, nil
82+
}

‎test/extended/util/framework.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2403,3 +2403,26 @@ func IsCapabilityEnabled(oc *CLI, cap configv1.ClusterVersionCapability) (bool,
24032403
}
24042404
return false, nil
24052405
}
2406+
2407+
// SkipIfNotPlatform skip the test if supported platforms are not matched
2408+
func SkipIfNotPlatform(oc *CLI, platforms ...configv1.PlatformType) {
2409+
var match bool
2410+
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
2411+
o.Expect(err).NotTo(o.HaveOccurred())
2412+
for _, platform := range platforms {
2413+
if infra.Status.PlatformStatus.Type == platform {
2414+
match = true
2415+
break
2416+
}
2417+
}
2418+
if !match {
2419+
g.Skip("Skip this test scenario because it is not supported on the " + string(infra.Status.PlatformStatus.Type) + " platform")
2420+
}
2421+
}
2422+
2423+
// GetClusterRegion get the cluster's region
2424+
func GetClusterRegion(oc *CLI) string {
2425+
region, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", `-ojsonpath={.items[].metadata.labels.topology\.kubernetes\.io/region}`).Output()
2426+
o.Expect(err).NotTo(o.HaveOccurred())
2427+
return region
2428+
}

0 commit comments

Comments
 (0)