Merge pull request #1 from vllm-project/reasoning-mode

feat: support auto-enable reasoning mode based on intention

Author: Xunzhuo

config/config.yaml

@@ -31,7 +31,7 @@ static_resources:
                   upstream_local_address: "%UPSTREAM_LOCAL_ADDRESS%"
                   request_id: "%REQ(X-REQUEST-ID)%"
                   selected_model: "%REQ(X-SELECTED-MODEL)%"
-                  selected_endpoint: "%REQ(X-GATEWAY-DESTINATION-ENDPOINT)%"
+                  selected_endpoint: "%REQ(X-SEMANTIC-DESTINATION-ENDPOINT)%"
           route_config:
             name: local_route
             virtual_hosts:
@@ -42,7 +42,7 @@ static_resources:
               - match:
                   prefix: "/"
                   headers:
-                  - name: "x-gateway-destination-endpoint"
+                  - name: "x-semantic-destination-endpoint"
                     string_match:
                       exact: "endpoint1"
                 route:
@@ -51,7 +51,7 @@ static_resources:
               - match:
                   prefix: "/"
                   headers:
-                  - name: "x-gateway-destination-endpoint"
+                  - name: "x-semantic-destination-endpoint"
                     string_match:
                       exact: "endpoint2"
                 route:
@@ -60,7 +60,7 @@ static_resources:
               - match:
                   prefix: "/"
                   headers:
-                  - name: "x-gateway-destination-endpoint"
+                  - name: "x-semantic-destination-endpoint"
                     string_match:
                       exact: "endpoint3"
                 route:

config/envoy.yaml

@@ -213,9 +213,11 @@ type ModelScore struct {
 }
 
 type Category struct {
-	Name        string       `yaml:"name"`
-	Description string       `yaml:"description,omitempty"`
-	ModelScores []ModelScore `yaml:"model_scores"`
+	Name                 string       `yaml:"name"`
+	Description          string       `yaml:"description,omitempty"`
+	UseReasoning         bool         `yaml:"use_reasoning"`
+	ReasoningDescription string       `yaml:"reasoning_description,omitempty"`
+	ModelScores          []ModelScore `yaml:"model_scores"`
 }
 
 var (

src/semantic-router/pkg/config/config.go

@@ -75,7 +75,7 @@ var _ = Describe("Endpoint Selection", func() {
 					var modelHeaderFound bool
 
 					for _, header := range headerMutation.SetHeaders {
-						if header.Header.Key == "x-gateway-destination-endpoint" {
+						if header.Header.Key == "x-semantic-destination-endpoint" {
 							endpointHeaderFound = true
 							// Should be one of the configured endpoints
 							Expect(header.Header.Value).To(BeElementOf("test-endpoint1", "test-endpoint2"))
@@ -139,7 +139,7 @@ var _ = Describe("Endpoint Selection", func() {
 					var selectedEndpoint string
 
 					for _, header := range headerMutation.SetHeaders {
-						if header.Header.Key == "x-gateway-destination-endpoint" {
+						if header.Header.Key == "x-semantic-destination-endpoint" {
 							endpointHeaderFound = true
 							selectedEndpoint = header.Header.Value
 							break
@@ -198,7 +198,7 @@ var _ = Describe("Endpoint Selection", func() {
 					var selectedEndpoint string
 
 					for _, header := range headerMutation.SetHeaders {
-						if header.Header.Key == "x-gateway-destination-endpoint" {
+						if header.Header.Key == "x-semantic-destination-endpoint" {
 							endpointHeaderFound = true
 							selectedEndpoint = header.Header.Value
 							break

src/semantic-router/pkg/extproc/endpoint_selection_test.go

@@ -52,22 +52,30 @@ func (r *OpenAIRouter) Process(stream ext_proc.ExternalProcessor_ProcessServer)
 
 		switch v := req.Request.(type) {
 		case *ext_proc.ProcessingRequest_RequestHeaders:
+			log.Printf("DEBUG: Processing request headers")
 			response, err := r.handleRequestHeaders(v, ctx)
 			if err != nil {
+				log.Printf("ERROR: handleRequestHeaders failed: %v", err)
 				return err
 			}
-			if err := sendResponse(stream, response, "header"); err != nil {
+			if err := sendResponse(stream, response, "request header"); err != nil {
+				log.Printf("ERROR: sendResponse for headers failed: %v", err)
 				return err
 			}
+			log.Printf("DEBUG: Request headers processed successfully")
 
 		case *ext_proc.ProcessingRequest_RequestBody:
+			log.Printf("DEBUG: Processing request body - THIS IS WHERE ROUTING HAPPENS")
 			response, err := r.handleRequestBody(v, ctx)
 			if err != nil {
+				log.Printf("ERROR: handleRequestBody failed: %v", err)
 				return err
 			}
-			if err := sendResponse(stream, response, "body"); err != nil {
+			if err := sendResponse(stream, response, "request body"); err != nil {
+				log.Printf("ERROR: sendResponse for body failed: %v", err)
 				return err
 			}
+			log.Printf("DEBUG: Request body processed successfully")
 
 		case *ext_proc.ProcessingRequest_ResponseHeaders:
 			response, err := r.handleResponseHeaders(v)
@@ -105,5 +113,7 @@ func (r *OpenAIRouter) Process(stream ext_proc.ExternalProcessor_ProcessServer)
 				return err
 			}
 		}
+
+		log.Printf("DEBUG: Finished processing message, continuing to next...")
 	}
 }