update

Signed-off-by: bitliu <[email protected]>

Author: Xunzhuo

deploy/kubernetes/aibrix/aigw-resources/gwapi-resources.yaml

@@ -19,7 +19,7 @@ spec:
               authority: semantic-router.vllm-semantic-router-system:50051
               clusterName: semantic-router
             timeout: 60s
-          message_timeout: 10s
+          message_timeout: 60s
           processing_mode:
             request_body_mode: BUFFERED
             request_header_mode: SEND
@@ -33,7 +33,7 @@ spec:
       op: add
       path: ''
       value:
-        connect_timeout: 10s
+        connect_timeout: 60s
         http2_protocol_options: {}
         lb_policy: ROUND_ROBIN
         load_assignment:

docs/default-model-fallback.md

@@ -0,0 +1,137 @@
+# Default Model Fallback Implementation
+
+## Problem
+
+When classification fails to identify a category (e.g., empty category name), the system was not selecting any model, leaving `selectedModel` empty. This could cause issues downstream in the request processing pipeline.
+
+### Example Scenario
+```
+User Request: "Hello" (generic greeting)
+Classification Result: category="" (no specific category matched)
+Selected Model: "" (empty - no model selected!)
+```
+
+## Solution
+
+Implemented a fallback mechanism to use the configured `default_model` when no category is classified.
+
+### Changes Made
+
+#### Modified `performClassificationAndModelSelection` in `req_filter_classification.go`
+
+**Before:**
+```go
+// Select best model for this category
+if categoryName != "" {
+    selectedModel = r.Classifier.SelectBestModelForCategory(categoryName)
+    logging.Infof("Selected model for category %s: %s", categoryName, selectedModel)
+} else {
+    // Empty else block - no model selected!
+}
+
+return categoryName, classificationConfidence, reasoningDecision, selectedModel
+```
+
+**After:**
+```go
+// Select best model for this category
+if categoryName != "" {
+    selectedModel = r.Classifier.SelectBestModelForCategory(categoryName)
+    logging.Infof("Selected model for category %s: %s", categoryName, selectedModel)
+} else {
+    // No category found, use default model
+    selectedModel = r.Config.DefaultModel
+    logging.Infof("No category classified, using default model: %s", selectedModel)
+}
+
+return categoryName, classificationConfidence, reasoningDecision, selectedModel
+```
+
+## Behavior
+
+### Before Fix
+| Scenario | Category | Selected Model | Issue |
+|----------|----------|----------------|-------|
+| Specific query ("Write Python code") | `coding` | `coding-expert-model` | ✅ Works |
+| Generic query ("Hello") | `` (empty) | `` (empty) | ❌ **No model selected** |
+| Classification error | `` (empty) | `` (empty) | ❌ **No model selected** |
+
+### After Fix
+| Scenario | Category | Selected Model | Status |
+|----------|----------|----------------|--------|
+| Specific query ("Write Python code") | `coding` | `coding-expert-model` | ✅ Works |
+| Generic query ("Hello") | `` (empty) | `default-model` | ✅ **Fallback to default** |
+| Classification error | `` (empty) | `default-model` | ✅ **Fallback to default** |
+
+## Configuration
+
+The default model is configured in the router configuration:
+
+```yaml
+# config.yaml
+default_model: "base-model"  # This model will be used when no category is classified
+
+categories:
+  - name: coding
+    model_scores:
+      - model: base-model
+        lora_name: coding-expert
+        score: 0.9
+  
+  - name: math
+    model_scores:
+      - model: base-model
+        lora_name: math-expert
+        score: 0.9
+```
+
+## Impact
+
+- **Robustness**: System always has a model selected, even when classification fails
+- **User Experience**: Generic queries are handled gracefully instead of failing
+- **Backward Compatibility**: Existing configurations continue to work
+- **Logging**: Clear log messages indicate when default model is used
+
+## Related Components
+
+This change works in conjunction with:
+1. **PII Policy Checker**: Default model's PII policy will be checked
+2. **Model Selection**: Default model must be configured in `model_config`
+3. **LoRA Fallback**: If default model uses LoRA, it will inherit base model's PII policy (see `pii-lora-fallback-implementation.md`)
+
+## Example Logs
+
+### When category is classified:
+```
+{"level":"info","msg":"Classification Result: category=coding, confidence=0.950, reasoning=true"}
+{"level":"info","msg":"Selected model for category coding: coding-expert"}
+```
+
+### When no category is classified (new behavior):
+```
+{"level":"info","msg":"Classification Result: category=, confidence=0.000, reasoning=false"}
+{"level":"info","msg":"No category classified, using default model: base-model"}
+```
+
+## Files Modified
+
+1. `src/semantic-router/pkg/extproc/req_filter_classification.go` - Added default model fallback logic
+
+## Testing
+
+To test this behavior:
+
+1. Send a generic query that doesn't match any category:
+   ```bash
+   curl -X POST http://localhost:8080/v1/chat/completions \
+     -H "Content-Type: application/json" \
+     -d '{
+       "model": "MoM",
+       "messages": [{"role": "user", "content": "Hello"}]
+     }'
+   ```
+
+2. Check logs for "No category classified, using default model" message
+
+3. Verify the request is processed with the default model
+

src/semantic-router/pkg/extproc/processor_req_body.go

@@ -116,7 +116,7 @@ func (r *OpenAIRouter) handleModelRouting(openAIRequest *openai.ChatCompletionNe
 
 	isAutoModel := r.Config != nil && r.Config.IsAutoModelName(originalModel)
 
-	if isAutoModel && categoryName != "" && selectedModel != "" {
+	if isAutoModel && selectedModel != "" {
 		return r.handleAutoModelRouting(openAIRequest, originalModel, categoryName, reasoningDecision, selectedModel, ctx, response)
 	} else if !isAutoModel {
 		return r.handleSpecifiedModelRouting(openAIRequest, originalModel, ctx)
@@ -253,6 +253,9 @@ func (r *OpenAIRouter) modifyRequestBodyForAutoRouting(openAIRequest *openai.Cha
 		return nil, status.Errorf(codes.Internal, "error serializing modified request: %v", err)
 	}
 
+	if categoryName == "" {
+		return modifiedBody, nil
+	}
 	// Set reasoning mode
 	modifiedBody, err = r.setReasoningModeToRequestBody(modifiedBody, useReasoning, categoryName)
 	if err != nil {

src/semantic-router/pkg/extproc/req_filter_classification.go

@@ -55,6 +55,10 @@ func (r *OpenAIRouter) performClassificationAndModelSelection(originalModel stri
 	if categoryName != "" {
 		selectedModel = r.Classifier.SelectBestModelForCategory(categoryName)
 		logging.Infof("Selected model for category %s: %s", categoryName, selectedModel)
+	} else {
+		// No category found, use default model
+		selectedModel = r.Config.DefaultModel
+		logging.Infof("No category classified, using default model: %s", selectedModel)
 	}
 
 	return categoryName, classificationConfidence, reasoningDecision, selectedModel

src/semantic-router/pkg/extproc/req_filter_pii.go

@@ -40,6 +40,8 @@ func (r *OpenAIRouter) isPIIDetectionEnabled(categoryName string) bool {
 	piiThreshold := float32(0.0)
 	if categoryName != "" && r.Config != nil {
 		piiThreshold = r.Config.GetPIIThresholdForCategory(categoryName)
+	} else {
+		piiThreshold = r.Config.PIIModel.Threshold
 	}
 
 	if piiThreshold == 0.0 {