Skip to content

Commit 56cc1c4

Browse files
sarthypartysarthyparty
authored
Add GRPCRoute weighted backendRefs test (#3962)
* Add GRPCRoute weighted backendRefs test * fix verify * fix verify again
1 parent 180e20f commit 56cc1c4

File tree

11 files changed

+541
-232
lines changed

11 files changed

+541
-232
lines changed

‎conformance/tests/grpcroute-weight.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package tests
18+
19+
import (
20+
"fmt"
21+
"testing"
22+
23+
"google.golang.org/grpc/codes"
24+
"k8s.io/apimachinery/pkg/types"
25+
26+
v1 "sigs.k8s.io/gateway-api/apis/v1"
27+
pb "sigs.k8s.io/gateway-api/conformance/echo-basic/grpcechoserver"
28+
"sigs.k8s.io/gateway-api/conformance/utils/grpc"
29+
"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
30+
"sigs.k8s.io/gateway-api/conformance/utils/suite"
31+
"sigs.k8s.io/gateway-api/conformance/utils/weight"
32+
"sigs.k8s.io/gateway-api/pkg/features"
33+
)
34+
35+
func init() {
36+
ConformanceTests = append(ConformanceTests, GRPCRouteWeight)
37+
}
38+
39+
var GRPCRouteWeight = suite.ConformanceTest{
40+
ShortName: "GRPCRouteWeight",
41+
Description: "An GRPCRoute with weighted backends",
42+
Manifests: []string{"tests/grpcroute-weight.yaml"},
43+
Features: []features.FeatureName{
44+
features.SupportGateway,
45+
features.SupportGRPCRoute,
46+
},
47+
Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
48+
var (
49+
ns = "gateway-conformance-infra"
50+
routeNN = types.NamespacedName{Name: "weighted-backends", Namespace: ns}
51+
gwNN = types.NamespacedName{Name: "same-namespace", Namespace: ns}
52+
gwAddr = kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, true, routeNN)
53+
)
54+
55+
t.Run("Requests should have a distribution that matches the weight", func(t *testing.T) {
56+
expected := grpc.ExpectedResponse{
57+
EchoRequest: &pb.EchoRequest{},
58+
Response: grpc.Response{Code: codes.OK},
59+
Namespace: "gateway-conformance-infra",
60+
}
61+
62+
// Assert request succeeds before doing our distribution check
63+
grpc.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.GRPCClient, suite.TimeoutConfig, gwAddr, expected)
64+
65+
expectedWeights := map[string]float64{
66+
"grpc-infra-backend-v1": 0.7,
67+
"grpc-infra-backend-v2": 0.3,
68+
"grpc-infra-backend-v3": 0.0,
69+
}
70+
71+
sender := weight.NewFunctionBasedSender(func() (string, error) {
72+
uniqueExpected := expected
73+
if err := grpc.AddEntropy(&uniqueExpected); err != nil {
74+
return "", fmt.Errorf("error adding entropy: %w", err)
75+
}
76+
client := &grpc.DefaultClient{}
77+
defer client.Close()
78+
resp, err := client.SendRPC(t, gwAddr, uniqueExpected, suite.TimeoutConfig.MaxTimeToConsistency)
79+
if err != nil {
80+
return "", fmt.Errorf("failed to send gRPC request: %w", err)
81+
}
82+
if resp.Code != codes.OK {
83+
return "", fmt.Errorf("expected OK response, got %v", resp.Code)
84+
}
85+
return resp.Response.GetAssertions().GetContext().GetPod(), nil
86+
})
87+
88+
for i := 0; i < weight.MaxTestRetries; i++ {
89+
if err := weight.TestWeightedDistribution(sender, expectedWeights); err != nil {
90+
t.Logf("Traffic distribution test failed (%d/%d): %s", i+1, weight.MaxTestRetries, err)
91+
} else {
92+
return
93+
}
94+
}
95+
t.Fatal("Weighted distribution tests failed")
96+
})
97+
},
98+
}

‎conformance/tests/grpcroute-weight.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: gateway.networking.k8s.io/v1
2+
kind: GRPCRoute
3+
metadata:
4+
name: weighted-backends
5+
namespace: gateway-conformance-infra
6+
spec:
7+
parentRefs:
8+
- name: same-namespace
9+
rules:
10+
- backendRefs:
11+
- name: grpc-infra-backend-v1
12+
port: 8080
13+
weight: 70
14+
- name: grpc-infra-backend-v2
15+
port: 8080
16+
weight: 30
17+
- name: grpc-infra-backend-v3
18+
port: 8080
19+
weight: 0

‎conformance/tests/httproute-weight.go

Lines changed: 26 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,15 @@ limitations under the License.
1717
package tests
1818

1919
import (
20-
"cmp"
21-
"errors"
2220
"fmt"
23-
"math"
24-
"slices"
25-
"strings"
26-
"sync"
2721
"testing"
2822

29-
"golang.org/x/sync/errgroup"
3023
"k8s.io/apimachinery/pkg/types"
3124

3225
"sigs.k8s.io/gateway-api/conformance/utils/http"
3326
"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
3427
"sigs.k8s.io/gateway-api/conformance/utils/suite"
28+
"sigs.k8s.io/gateway-api/conformance/utils/weight"
3529
"sigs.k8s.io/gateway-api/pkg/features"
3630
)
3731

@@ -69,9 +63,31 @@ var HTTPRouteWeight = suite.ConformanceTest{
6963
// Assert request succeeds before doing our distribution check
7064
http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expected)
7165

72-
for i := 0; i < 10; i++ {
73-
if err := testDistribution(t, suite, gwAddr, expected); err != nil {
74-
t.Logf("Traffic distribution test failed (%d/10): %s", i+1, err)
66+
expectedWeights := map[string]float64{
67+
"infra-backend-v1": 0.7,
68+
"infra-backend-v2": 0.3,
69+
"infra-backend-v3": 0.0,
70+
}
71+
72+
sender := weight.NewFunctionBasedSender(func() (string, error) {
73+
uniqueExpected := expected
74+
if err := http.AddEntropy(&uniqueExpected); err != nil {
75+
return "", fmt.Errorf("error adding entropy: %w", err)
76+
}
77+
req := http.MakeRequest(t, &uniqueExpected, gwAddr, "HTTP", "http")
78+
cReq, cRes, err := suite.RoundTripper.CaptureRoundTrip(req)
79+
if err != nil {
80+
return "", fmt.Errorf("failed to roundtrip request: %w", err)
81+
}
82+
if err := http.CompareRoundTrip(t, &req, cReq, cRes, expected); err != nil {
83+
return "", fmt.Errorf("response expectation failed for request: %w", err)
84+
}
85+
return cReq.Pod, nil
86+
})
87+
88+
for i := 0; i < weight.MaxTestRetries; i++ {
89+
if err := weight.TestWeightedDistribution(sender, expectedWeights); err != nil {
90+
t.Logf("Traffic distribution test failed (%d/%d): %s", i+1, weight.MaxTestRetries, err)
7591
} else {
7692
return
7793
}
@@ -80,85 +96,3 @@ var HTTPRouteWeight = suite.ConformanceTest{
8096
})
8197
},
8298
}
83-
84-
func testDistribution(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr string, expected http.ExpectedResponse) error {
85-
const (
86-
concurrentRequests = 10
87-
tolerancePercentage = 0.05
88-
totalRequests = 500.0
89-
)
90-
var (
91-
roundTripper = suite.RoundTripper
92-
93-
g errgroup.Group
94-
seenMutex sync.Mutex
95-
seen = make(map[string]float64, 3 /* number of backends */)
96-
expectedWeights = map[string]float64{
97-
"infra-backend-v1": 0.7,
98-
"infra-backend-v2": 0.3,
99-
"infra-backend-v3": 0.0,
100-
}
101-
)
102-
g.SetLimit(concurrentRequests)
103-
for i := 0.0; i < totalRequests; i++ {
104-
g.Go(func() error {
105-
uniqueExpected := expected
106-
if err := http.AddEntropy(&uniqueExpected); err != nil {
107-
return fmt.Errorf("error adding entropy: %w", err)
108-
}
109-
req := http.MakeRequest(t, &uniqueExpected, gwAddr, "HTTP", "http")
110-
cReq, cRes, err := roundTripper.CaptureRoundTrip(req)
111-
if err != nil {
112-
return fmt.Errorf("failed to roundtrip request: %w", err)
113-
}
114-
if err := http.CompareRoundTrip(t, &req, cReq, cRes, expected); err != nil {
115-
return fmt.Errorf("response expectation failed for request: %w", err)
116-
}
117-
118-
seenMutex.Lock()
119-
defer seenMutex.Unlock()
120-
121-
for expectedBackend := range expectedWeights {
122-
if strings.HasPrefix(cReq.Pod, expectedBackend) {
123-
seen[expectedBackend]++
124-
return nil
125-
}
126-
}
127-
128-
return fmt.Errorf("request was handled by an unexpected pod %q", cReq.Pod)
129-
})
130-
}
131-
132-
if err := g.Wait(); err != nil {
133-
return fmt.Errorf("error while sending requests: %w", err)
134-
}
135-
136-
var errs []error
137-
if len(seen) != 2 {
138-
errs = append(errs, fmt.Errorf("expected only two backends to receive traffic"))
139-
}
140-
141-
for wantBackend, wantPercent := range expectedWeights {
142-
gotCount, ok := seen[wantBackend]
143-
144-
if !ok && wantPercent != 0.0 {
145-
errs = append(errs, fmt.Errorf("expect traffic to hit backend %q - but none was received", wantBackend))
146-
continue
147-
}
148-
149-
gotPercent := gotCount / totalRequests
150-
151-
if math.Abs(gotPercent-wantPercent) > tolerancePercentage {
152-
errs = append(errs, fmt.Errorf("backend %q weighted traffic of %v not within tolerance %v (+/-%f)",
153-
wantBackend,
154-
gotPercent,
155-
wantPercent,
156-
tolerancePercentage,
157-
))
158-
}
159-
}
160-
slices.SortFunc(errs, func(a, b error) int {
161-
return cmp.Compare(a.Error(), b.Error())
162-
})
163-
return errors.Join(errs...)
164-
}

‎conformance/tests/mesh/grpcroute-weight.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package meshtests
18+
19+
import (
20+
"fmt"
21+
"testing"
22+
23+
"sigs.k8s.io/gateway-api/conformance/utils/echo"
24+
"sigs.k8s.io/gateway-api/conformance/utils/http"
25+
"sigs.k8s.io/gateway-api/conformance/utils/suite"
26+
"sigs.k8s.io/gateway-api/conformance/utils/weight"
27+
"sigs.k8s.io/gateway-api/pkg/features"
28+
)
29+
30+
func init() {
31+
MeshConformanceTests = append(MeshConformanceTests, MeshGRPCRouteWeight)
32+
}
33+
34+
var MeshGRPCRouteWeight = suite.ConformanceTest{
35+
ShortName: "MeshGRPCRouteWeight",
36+
Description: "A GRPCRoute with weighted backends in mesh mode",
37+
Manifests: []string{"tests/mesh/grpcroute-weight.yaml"},
38+
Features: []features.FeatureName{
39+
features.SupportMesh,
40+
features.SupportGRPCRoute,
41+
},
42+
Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
43+
client := echo.ConnectToApp(t, s, echo.MeshAppEchoV1)
44+
45+
t.Run("Requests should have a distribution that matches the weight", func(t *testing.T) {
46+
// Create a gRPC request using the mesh client framework
47+
expected := http.ExpectedResponse{
48+
Request: http.Request{Protocol: "grpc", Path: "", Host: "echo:7070"},
49+
Response: http.Response{StatusCode: 200},
50+
Namespace: "gateway-conformance-mesh",
51+
}
52+
53+
// Assert request succeeds before doing our distribution check
54+
client.MakeRequestAndExpectEventuallyConsistentResponse(t, expected, s.TimeoutConfig)
55+
56+
expectedWeights := map[string]float64{
57+
"echo-v1": 0.7,
58+
"echo-v2": 0.3,
59+
}
60+
61+
sender := weight.NewFunctionBasedSender(func() (string, error) {
62+
uniqueExpected := expected
63+
if err := http.AddEntropy(&uniqueExpected); err != nil {
64+
return "", fmt.Errorf("error adding entropy: %w", err)
65+
}
66+
_, cRes, err := client.CaptureRequestResponseAndCompare(t, uniqueExpected)
67+
if err != nil {
68+
return "", fmt.Errorf("failed gRPC mesh request: %w", err)
69+
}
70+
return cRes.Hostname, nil
71+
})
72+
73+
for i := 0; i < weight.MaxTestRetries; i++ {
74+
if err := weight.TestWeightedDistribution(sender, expectedWeights); err != nil {
75+
t.Logf("Traffic distribution test failed (%d/%d): %s", i+1, weight.MaxTestRetries, err)
76+
} else {
77+
return
78+
}
79+
}
80+
t.Fatal("Weighted distribution tests failed")
81+
})
82+
},
83+
}

‎conformance/tests/mesh/grpcroute-weight.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: gateway.networking.k8s.io/v1
2+
kind: GRPCRoute
3+
metadata:
4+
name: mesh-grpc-weighted-backends
5+
namespace: gateway-conformance-mesh
6+
spec:
7+
parentRefs:
8+
- group: ""
9+
kind: Service
10+
name: echo
11+
port: 7070
12+
rules:
13+
- backendRefs:
14+
- name: echo-v1
15+
port: 7070
16+
weight: 70
17+
- name: echo-v2
18+
port: 7070
19+
weight: 30
20+
- name: echo-v3
21+
port: 7070
22+
weight: 0

0 commit comments

Comments
 (0)