|
| 1 | +/* |
| 2 | +Copyright 2025 The Kubernetes Authors. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package basic |
| 18 | + |
| 19 | +import ( |
| 20 | + "fmt" |
| 21 | + "net/http" |
| 22 | + "slices" |
| 23 | + "strings" |
| 24 | + "testing" |
| 25 | + |
| 26 | + "github.com/stretchr/testify/require" |
| 27 | + "golang.org/x/sync/errgroup" |
| 28 | + "k8s.io/apimachinery/pkg/types" |
| 29 | + "sigs.k8s.io/gateway-api/conformance/utils/suite" |
| 30 | + "sigs.k8s.io/gateway-api/pkg/features" |
| 31 | + |
| 32 | + "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" |
| 33 | + k8sutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes" |
| 34 | + "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic" |
| 35 | + trafficutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic" |
| 36 | + gwhttp "sigs.k8s.io/gateway-api/conformance/utils/http" |
| 37 | +) |
| 38 | + |
| 39 | +func init() { |
| 40 | + // Register the GatewayFollowingEPPRouting test case with the conformance suite. |
| 41 | + // This ensures it will be discovered and run by the test runner. |
| 42 | + tests.ConformanceTests = append(tests.ConformanceTests, GatewayFollowingEPPRouting) |
| 43 | +} |
| 44 | + |
| 45 | +// GatewayFollowingEPPRouting defines the test case for verifying gateway should send traffic to an endpoint in the list returned by EPP. |
| 46 | +var GatewayFollowingEPPRouting = suite.ConformanceTest{ |
| 47 | + ShortName: "GatewayFollowingEPPRouting", |
| 48 | + Description: "Inference gateway should send traffic to an endpoint in the list returned by EPP", |
| 49 | + Manifests: []string{"tests/basic/gateway_following_epp_routing.yaml"}, |
| 50 | + Features: []features.FeatureName{ |
| 51 | + features.FeatureName("SupportInferencePool"), |
| 52 | + features.SupportGateway, |
| 53 | + }, |
| 54 | + Test: func(t *testing.T, s *suite.ConformanceTestSuite) { |
| 55 | + const ( |
| 56 | + appBackendNamespace = "gateway-conformance-app-backend" |
| 57 | + infraNamespace = "gateway-conformance-infra" |
| 58 | + hostname = "primary.example.com" |
| 59 | + path = "/primary-gateway-test" |
| 60 | + expectedPodReplicas = 3 |
| 61 | + // eppSelectionHeaderName is the custom header used by the testing-EPP service |
| 62 | + // to determine which endpoint to select. |
| 63 | + eppSelectionHeaderName = "test-epp-endpoint-selection" |
| 64 | + appPodBackendPrefix = "primary-inference-model-server" |
| 65 | + ) |
| 66 | + |
| 67 | + httpRouteNN := types.NamespacedName{Name: "httproute-for-primary-gw", Namespace: appBackendNamespace} |
| 68 | + gatewayNN := types.NamespacedName{Name: "conformance-primary-gateway", Namespace: infraNamespace} |
| 69 | + poolNN := types.NamespacedName{Name: "primary-inference-pool", Namespace: appBackendNamespace} |
| 70 | + backendPodLabels := map[string]string{"app": "primary-inference-model-server"} |
| 71 | + |
| 72 | + t.Log("Verifying HTTPRoute and InferencePool are accepted and the Gateway has an address.") |
| 73 | + k8sutils.HTTPRouteMustBeAcceptedAndResolved(t, s.Client, s.TimeoutConfig, httpRouteNN, gatewayNN) |
| 74 | + k8sutils.InferencePoolMustBeAcceptedByParent(t, s.Client, poolNN) |
| 75 | + gwAddr := k8sutils.GetGatewayEndpoint(t, s.Client, s.TimeoutConfig, gatewayNN) |
| 76 | + |
| 77 | + t.Logf("Fetching backend pods with labels: %v", backendPodLabels) |
| 78 | + pods, err := k8sutils.GetPodsWithLabel(t, s.Client, appBackendNamespace, backendPodLabels) |
| 79 | + require.NoError(t, err, "Failed to get backend pods") |
| 80 | + require.Len(t, pods, expectedPodReplicas, "Expected to find %d backend pods, but found %d.", expectedPodReplicas, len(pods)) |
| 81 | + |
| 82 | + podIPs := make([]string, len(pods)) |
| 83 | + podNames := make([]string, len(pods)) |
| 84 | + for i, pod := range pods { |
| 85 | + podIPs[i] = pod.Status.PodIP |
| 86 | + podNames[i] = pod.Name |
| 87 | + } |
| 88 | + |
| 89 | + requestBody := `{ |
| 90 | + "model": "conformance-fake-model", |
| 91 | + "prompt": "Write as if you were a critic: San Francisco" |
| 92 | + }` |
| 93 | + |
| 94 | + for i := 0; i < len(pods); i++ { |
| 95 | + // Send an initial request targeting a single pod and wait for it to be successful to ensure the Gateway and EPP |
| 96 | + // are functioning correctly before running the main test cases. |
| 97 | + trafficutils.MakeRequestWithRequestParamAndExpectSuccess( |
| 98 | + t, |
| 99 | + s.RoundTripper, |
| 100 | + s.TimeoutConfig, |
| 101 | + gwAddr, |
| 102 | + trafficutils.Request{ |
| 103 | + Host: hostname, |
| 104 | + Path: path, |
| 105 | + Headers: map[string]string{eppSelectionHeaderName: podIPs[i]}, |
| 106 | + Method: http.MethodPost, |
| 107 | + Body: requestBody, |
| 108 | + Backend: podNames[i], |
| 109 | + Namespace: appBackendNamespace, |
| 110 | + }, |
| 111 | + ) |
| 112 | + } |
| 113 | + |
| 114 | + testCases := []struct { |
| 115 | + name string |
| 116 | + podIPsToBeReturnedByEPP []string |
| 117 | + expectAllRequestsRoutedWithinPodNames []string |
| 118 | + }{ |
| 119 | + { |
| 120 | + name: "should route traffic to a single designated pod", |
| 121 | + podIPsToBeReturnedByEPP: []string{podIPs[2]}, |
| 122 | + expectAllRequestsRoutedWithinPodNames: []string{podNames[2]}, |
| 123 | + }, |
| 124 | + { |
| 125 | + name: "should route traffic to two designated pods", |
| 126 | + podIPsToBeReturnedByEPP: []string{podIPs[0], podIPs[1]}, |
| 127 | + expectAllRequestsRoutedWithinPodNames: []string{podNames[0], podNames[1]}, |
| 128 | + }, |
| 129 | + { |
| 130 | + name: "should route traffic to all available pods", |
| 131 | + podIPsToBeReturnedByEPP: []string{podIPs[0], podIPs[1], podIPs[2]}, |
| 132 | + expectAllRequestsRoutedWithinPodNames: []string{podNames[0], podNames[1], podNames[2]}, |
| 133 | + }, |
| 134 | + } |
| 135 | + |
| 136 | + for _, tc := range testCases { |
| 137 | + t.Run(tc.name, func(t *testing.T) { |
| 138 | + eppHeaderValue := strings.Join(tc.podIPsToBeReturnedByEPP, ",") |
| 139 | + headers := map[string]string{eppSelectionHeaderName: eppHeaderValue} |
| 140 | + |
| 141 | + t.Logf("Sending request to %s with EPP header '%s: %s'", gwAddr, eppSelectionHeaderName, eppHeaderValue) |
| 142 | + t.Logf("Expecting traffic to be routed to pod: %v", tc.expectAllRequestsRoutedWithinPodNames) |
| 143 | + |
| 144 | + assertTrafficOnlyReachesToExpectedPods(t, s, gwAddr, gwhttp.ExpectedResponse{ |
| 145 | + Request: gwhttp.Request{ |
| 146 | + Host: hostname, |
| 147 | + Path: path, |
| 148 | + Method: http.MethodPost, |
| 149 | + Headers: headers, |
| 150 | + }, |
| 151 | + Response: gwhttp.Response{ |
| 152 | + StatusCode: http.StatusOK, |
| 153 | + }, |
| 154 | + Backend: appPodBackendPrefix, |
| 155 | + Namespace: appBackendNamespace, |
| 156 | + }, requestBody, tc.expectAllRequestsRoutedWithinPodNames) |
| 157 | + }) |
| 158 | + } |
| 159 | + }, |
| 160 | +} |
| 161 | + |
| 162 | +func assertTrafficOnlyReachesToExpectedPods(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr string, expected gwhttp.ExpectedResponse, requestBody string, expectedPodNames []string) { |
| 163 | + t.Helper() |
| 164 | + const ( |
| 165 | + concurrentRequests = 10 |
| 166 | + totalRequests = 100 |
| 167 | + ) |
| 168 | + var ( |
| 169 | + roundTripper = suite.RoundTripper |
| 170 | + g errgroup.Group |
| 171 | + req = gwhttp.MakeRequest(t, &expected, gwAddr, "HTTP", "http") |
| 172 | + ) |
| 173 | + g.SetLimit(concurrentRequests) |
| 174 | + for i := 0; i < totalRequests; i++ { |
| 175 | + g.Go(func() error { |
| 176 | + cReq, cRes, err := traffic.MakeCallRoundTripper(t, roundTripper, &traffic.RequestWithBody{Request: req, Body: strings.NewReader(requestBody)}) |
| 177 | + if err != nil { |
| 178 | + return fmt.Errorf("failed to roundtrip request: %w", err) |
| 179 | + } |
| 180 | + if err := gwhttp.CompareRequest(t, &req, cReq, cRes, expected); err != nil { |
| 181 | + return fmt.Errorf("response expectation failed for request: %w", err) |
| 182 | + } |
| 183 | + |
| 184 | + if slices.Contains(expectedPodNames, cReq.Pod) { |
| 185 | + return nil |
| 186 | + } |
| 187 | + return fmt.Errorf("request was handled by an unexpected pod %q", cReq.Pod) |
| 188 | + }) |
| 189 | + } |
| 190 | + if err := g.Wait(); err != nil { |
| 191 | + t.Fatalf("Not all the requests are sent to the expectedPods successfully, err: %v", err) |
| 192 | + } |
| 193 | + t.Logf("Traffic successfully reached only to expected pods: %v", expectedPodNames) |
| 194 | +} |
0 commit comments