update

Signed-off-by: bitliu <[email protected]>

Author: Xunzhuo

deploy/kubernetes/ai-gateway/aigw-resources/gwapi-resources.yaml

@@ -38,6 +38,21 @@ spec:
       kind: EnvoyProxy
       name: semantic-router
 ---
+# By default, Envoy Gateway sets the buffer limit to 32kiB which is not sufficient for AI workloads.
+# This ClientTrafficPolicy sets the buffer limit to 50MiB as an example.
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: ClientTrafficPolicy
+metadata:
+  name: semantic-router
+  namespace: default
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: semantic-router
+  connection:
+    bufferLimit: 50Mi
+---
 apiVersion: gateway.envoyproxy.io/v1alpha1
 kind: ClientTrafficPolicy
 metadata:
@@ -69,41 +84,59 @@ spec:
         value: math-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
   - matches:
     - headers:
       - type: Exact
         name: x-ai-eg-model
         value: science-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
   - matches:
     - headers:
       - type: Exact
         name: x-ai-eg-model
         value: social-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
   - matches:
     - headers:
       - type: Exact
         name: x-ai-eg-model
         value: humanities-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
   - matches:
     - headers:
       - type: Exact
         name: x-ai-eg-model
         value: law-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
   - matches:
     - headers:
       - type: Exact
         name: x-ai-eg-model
         value: general-expert
     backendRefs:
     - name: vllm-llama3-8b-instruct
+    timeouts:
+      request: 60s
+      backendRequest: 60s
 ---
 apiVersion: gateway.envoyproxy.io/v1alpha1
 kind: EnvoyPatchPolicy
@@ -126,7 +159,7 @@ spec:
               authority: semantic-router.vllm-semantic-router-system:50051
               clusterName: semantic-router
             timeout: 60s
-          message_timeout: 10s
+          message_timeout: 60s
           processing_mode:
             request_body_mode: BUFFERED
             request_header_mode: SEND
@@ -140,7 +173,7 @@ spec:
       op: add
       path: ''
       value:
-        connect_timeout: 10s
+        connect_timeout: 60s
         http2_protocol_options: {}
         lb_policy: ROUND_ROBIN
         load_assignment: