Skip to content

Commit b70e22e

Browse files
AlinsRanAlinsRan
authored
fix: wss related tests are unstable (#2675)
1 parent 0c0c5f9 commit b70e22e

File tree

9 files changed

+88
-66
lines changed

9 files changed

+88
-66
lines changed

‎internal/controller/gateway_controller.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
140140
}
141141
return ctrl.Result{}, err
142142
}
143+
if !r.checkGatewayClass(gateway) {
144+
return ctrl.Result{}, nil
145+
}
146+
143147
conditionProgrammedStatus, conditionProgrammedMsg := true, "Programmed"
144148

145149
r.Log.Info("gateway has been accepted", "gateway", gateway.GetName())

‎internal/controller/indexer/ssl_host.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,11 @@ func GatewayTLSHostIndexFunc(rawObj client.Object) []string {
6969
}
7070
}
7171

72-
tlsHostIndexLogger.Info("GatewayTLSHostIndexFunc", "hosts", hostSetToSlice(hosts), "len", len(hostSetToSlice(hosts)))
72+
hostsSlice := hostSetToSlice(hosts)
7373

74-
return hostSetToSlice(hosts)
74+
tlsHostIndexLogger.V(1).Info("GatewayTLSHostIndexFunc", "hosts", hostsSlice)
75+
76+
return hostsSlice
7577
}
7678

7779
// IngressTLSHostIndexFunc indexes Ingresses by their TLS SNI hosts.

‎test/e2e/crds/v2/route.go‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2145,22 +2145,11 @@ spec:
21452145
time.Sleep(6 * time.Second)
21462146

21472147
By("verify wss connection")
2148-
u := url.URL{
2149-
Scheme: "wss",
2150-
Host: s.GetAPISIXHTTPSEndpoint(),
2151-
Path: "/ws",
2152-
}
2153-
headers := http.Header{"Host": []string{"api6.com"}}
2154-
dialer := websocket.Dialer{
2155-
TLSClientConfig: &tls.Config{
2156-
InsecureSkipVerify: true,
2157-
ServerName: "api6.com",
2158-
},
2159-
}
2160-
2161-
conn, resp, err := dialer.Dial(u.String(), headers)
2162-
Expect(err).ShouldNot(HaveOccurred(), "WebSocket handshake")
2163-
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
2148+
hostname := "api6.com"
2149+
conn := s.NewWebsocketClient(&tls.Config{
2150+
InsecureSkipVerify: true,
2151+
ServerName: hostname,
2152+
}, "/ws", http.Header{"Host": []string{hostname}})
21642153

21652154
defer func() {
21662155
_ = conn.Close()

‎test/e2e/framework/manifests/apisix.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,18 @@ metadata:
6666
name: apisix
6767
labels:
6868
app.kubernetes.io/name: apisix
69+
app: apisix
6970
spec:
7071
replicas: {{ default 1 .Replicas }}
7172
selector:
7273
matchLabels:
7374
app.kubernetes.io/name: apisix
75+
app: apisix
7476
template:
7577
metadata:
7678
labels:
7779
app.kubernetes.io/name: apisix
80+
app: apisix
7881
spec:
7982
initContainers:
8083
- name: config-setup

‎test/e2e/framework/manifests/nginx.yaml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,13 +124,19 @@ spec:
124124
containers:
125125
- livenessProbe:
126126
failureThreshold: 3
127-
initialDelaySeconds: 1
128-
periodSeconds: 5
127+
initialDelaySeconds: 10
128+
periodSeconds: 15
129129
successThreshold: 1
130130
httpGet:
131131
path: /healthz
132132
port: 80
133133
timeoutSeconds: 2
134+
readinessProbe:
135+
httpGet:
136+
path: /healthz
137+
port: 80
138+
initialDelaySeconds: 5
139+
periodSeconds: 5
134140
image: "openresty/openresty:1.27.1.2-4-bullseye-fat"
135141
imagePullPolicy: IfNotPresent
136142
name: nginx

‎test/e2e/gatewayapi/httproute.go‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"crypto/tls"
2323
"fmt"
2424
"net/http"
25-
"net/url"
2625
"strings"
2726
"time"
2827

@@ -2504,36 +2503,21 @@ spec:
25042503

25052504
It("WSS backend", func() {
25062505
s.ResourceApplied("HTTPRoute", "nginx-wss", fmt.Sprintf(httprouteWithWSS, s.Namespace()), 1)
2507-
time.Sleep(6 * time.Second)
25082506

25092507
By("verify wss connection")
2510-
u := url.URL{
2511-
Scheme: "wss",
2512-
Host: s.GetAPISIXHTTPSEndpoint(),
2513-
Path: "/ws",
2514-
}
2515-
headers := http.Header{"Host": []string{"api6.com"}}
2516-
25172508
hostname := "api6.com"
2518-
2519-
dialer := websocket.Dialer{
2520-
TLSClientConfig: &tls.Config{
2521-
InsecureSkipVerify: true,
2522-
ServerName: hostname,
2523-
},
2524-
}
2525-
2526-
conn, resp, err := dialer.Dial(u.String(), headers)
2527-
Expect(err).ShouldNot(HaveOccurred(), "WebSocket handshake")
2528-
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
2509+
conn := s.NewWebsocketClient(&tls.Config{
2510+
InsecureSkipVerify: true,
2511+
ServerName: hostname,
2512+
}, "/ws", http.Header{"Host": []string{hostname}})
25292513

25302514
defer func() {
25312515
_ = conn.Close()
25322516
}()
25332517

25342518
By("send and receive message through WebSocket")
25352519
testMessage := "hello, this is APISIX"
2536-
err = conn.WriteMessage(websocket.TextMessage, []byte(testMessage))
2520+
err := conn.WriteMessage(websocket.TextMessage, []byte(testMessage))
25372521
Expect(err).ShouldNot(HaveOccurred(), "writing WebSocket message")
25382522

25392523
// Then our echo

‎test/e2e/ingress/ingress.go‎

Lines changed: 11 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"encoding/base64"
2424
"fmt"
2525
"net/http"
26-
"net/url"
2726
"strings"
2827
"time"
2928

@@ -944,14 +943,13 @@ spec:
944943
By("create Ingress")
945944
err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingress, s.Namespace()), s.Namespace())
946945
Expect(err).NotTo(HaveOccurred(), "creating Ingress")
947-
time.Sleep(5 * time.Second)
948946

949947
By("verify Ingress works")
950-
s.NewAPISIXClient().
951-
GET("/get").
952-
WithHost("ingress.example.com").
953-
Expect().
954-
Status(200)
948+
s.RequestAssert(&scaffold.RequestAssert{
949+
Method: "GET",
950+
Host: "ingress.example.com",
951+
Check: scaffold.WithExpectedStatus(http.StatusOK),
952+
})
955953

956954
By("create additional gateway group to get new admin key")
957955
additionalGatewayGroupID, _, err = s.Deployer.CreateAdditionalGateway("gateway-proxy-update")
@@ -1070,33 +1068,21 @@ spec:
10701068
createSecret(s, _secretName)
10711069
By("create Ingress")
10721070
Expect(s.CreateResourceFromString(fmt.Sprintf(ingressWithWSS, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress")
1073-
time.Sleep(6 * time.Second)
10741071

10751072
By("verify wss connection")
1076-
u := url.URL{
1077-
Scheme: "wss",
1078-
Host: s.GetAPISIXHTTPSEndpoint(),
1079-
Path: "/ws",
1080-
}
1081-
headers := http.Header{"Host": []string{"api6.com"}}
1082-
dialer := websocket.Dialer{
1083-
TLSClientConfig: &tls.Config{
1084-
InsecureSkipVerify: true,
1085-
ServerName: "api6.com",
1086-
},
1087-
}
1088-
1089-
conn, resp, err := dialer.Dial(u.String(), headers)
1090-
Expect(err).ShouldNot(HaveOccurred(), "WebSocket handshake")
1091-
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
1073+
hostname := "api6.com"
1074+
conn := s.NewWebsocketClient(&tls.Config{
1075+
InsecureSkipVerify: true,
1076+
ServerName: hostname,
1077+
}, "/ws", http.Header{"Host": []string{hostname}})
10921078

10931079
defer func() {
10941080
_ = conn.Close()
10951081
}()
10961082

10971083
By("send and receive message through WebSocket")
10981084
testMessage := "hello, this is APISIX"
1099-
err = conn.WriteMessage(websocket.TextMessage, []byte(testMessage))
1085+
err := conn.WriteMessage(websocket.TextMessage, []byte(testMessage))
11001086
Expect(err).ShouldNot(HaveOccurred(), "writing WebSocket message")
11011087

11021088
// Then our echo

‎test/e2e/scaffold/apisix_deployer.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,19 @@ func (s *APISIXDeployer) AfterEach() {
111111
if output != "" {
112112
_, _ = fmt.Fprintln(GinkgoWriter, output)
113113
}
114+
115+
_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX logs:")
116+
output = s.GetDeploymentLogs("apisix")
117+
if output != "" {
118+
_, _ = fmt.Fprintln(GinkgoWriter, output)
119+
}
120+
if framework.ProviderType == framework.ProviderTypeAPISIXStandalone && s.adminTunnel != nil {
121+
client := NewClient("http", s.adminTunnel.Endpoint())
122+
reporter := &ErrorReporter{}
123+
body := client.GET("/apisix/admin/configs").WithHeader("X-API-KEY", s.AdminKey()).WithReporter(reporter).Expect().Body().Raw()
124+
_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX configs:")
125+
_, _ = fmt.Fprintln(GinkgoWriter, body)
126+
}
114127
}
115128

116129
// Delete all additional gateways

‎test/e2e/scaffold/scaffold.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030

3131
"github.com/api7/gopkg/pkg/log"
3232
"github.com/gavv/httpexpect/v2"
33+
"github.com/gorilla/websocket"
3334
"github.com/gruntwork-io/terratest/modules/k8s"
3435
"github.com/gruntwork-io/terratest/modules/testing"
3536
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
@@ -562,6 +563,40 @@ func (s *Scaffold) GetMetricsEndpoint() string {
562563
return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint())
563564
}
564565

566+
func (s *Scaffold) NewWebsocketClient(tls *tls.Config, path string, headers http.Header) *websocket.Conn {
567+
var host = s.ApisixHTTPEndpoint()
568+
var scheme = "ws"
569+
if tls != nil {
570+
scheme = "wss"
571+
host = s.GetAPISIXHTTPSEndpoint()
572+
}
573+
574+
dialer := websocket.Dialer{
575+
TLSClientConfig: tls,
576+
}
577+
578+
u := url.URL{
579+
Scheme: scheme,
580+
Host: host,
581+
Path: path,
582+
}
583+
var conn *websocket.Conn
584+
585+
s.RetryAssertion(func() error {
586+
c, resp, err := dialer.Dial(u.String(), headers)
587+
if err != nil {
588+
return err
589+
}
590+
if resp == nil || resp.StatusCode != http.StatusSwitchingProtocols {
591+
_ = c.Close()
592+
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
593+
}
594+
conn = c
595+
return nil
596+
}).ShouldNot(HaveOccurred(), "establishing websocket connection")
597+
return conn
598+
}
599+
565600
func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) {
566601
tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090)
567602
if err := tunnel.ForwardPortE(s.t); err != nil {

0 commit comments

Comments
 (0)