Merge branch 'main' into 0909-yuluo/docs

Author: yuluo-yx

.github/workflows/test-and-build.yml

@@ -1,4 +1,4 @@
-name: Run Test
+name: Test And Build
 
 on:
   schedule:

README.md

@@ -6,6 +6,7 @@
 [![Hugging Face](https://img.shields.io/badge/🤗%20Hugging%20Face-Community-yellow)](https://huggingface.co/LLM-Semantic-Router)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
 [![Crates.io](https://img.shields.io/crates/v/candle-semantic-router.svg)](https://crates.io/crates/candle-semantic-router)
+![](https://github.com/vllm-project/semantic-router/workflows/Test%20And%20Build/badge.svg)
 
 **📚 [Complete Documentation](https://vllm-semantic-router.com) | 🚀 [Quick Start](https://vllm-semantic-router.com/docs/getting-started/installation) | 📣 [Blog](https://vllm-semantic-router.com/blog/) | 📖 [API Reference](https://vllm-semantic-router.com/docs/api/router/)**
 

config/config.yaml

@@ -112,7 +112,6 @@ classifier:
     threshold: 0.7
     use_cpu: true
     pii_mapping_path: "models/pii_classifier_modernbert-base_presidio_token_model/pii_type_mapping.json"
-  load_aware: false
 categories:
 - name: business
   use_reasoning: false

deploy/kubernetes/config.yaml

@@ -78,7 +78,6 @@ classifier:
     threshold: 0.7
     use_cpu: true
     pii_mapping_path: "models/pii_classifier_modernbert-base_presidio_token_model/pii_type_mapping.json"
-  load_aware: false
 categories:
 - name: business
   model_scores:

src/semantic-router/pkg/config/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"sync"
 
 	"gopkg.in/yaml.v3"
@@ -33,7 +34,6 @@ type RouterConfig struct {
 			UseCPU         bool    `yaml:"use_cpu"`
 			PIIMappingPath string  `yaml:"pii_mapping_path"`
 		} `yaml:"pii_model"`
-		LoadAware bool `yaml:"load_aware"`
 	} `yaml:"classifier"`
 
 	// Categories for routing queries
@@ -439,14 +439,7 @@ func (c *RouterConfig) IsModelAllowedForPIIType(modelName string, piiType string
 	}
 
 	// If allow_by_default is false, only explicitly allowed PII types are permitted
-	for _, allowedPII := range policy.PIITypes {
-		if allowedPII == piiType {
-			return true
-		}
-	}
-
-	// PII type not found in allowed list and allow_by_default is false
-	return false
+	return slices.Contains(policy.PIITypes, piiType)
 }
 
 // IsModelAllowedForPIITypes checks if a model is allowed to process any of the given PII types
@@ -487,23 +480,17 @@ func (c *RouterConfig) GetEndpointsForModel(modelName string) []VLLMEndpoint {
 
 	// First, find all endpoints that can serve this model
 	for _, endpoint := range c.VLLMEndpoints {
-		for _, model := range endpoint.Models {
-			if model == modelName {
-				availableEndpoints = append(availableEndpoints, endpoint)
-				break
-			}
+		if slices.Contains(endpoint.Models, modelName) {
+			availableEndpoints = append(availableEndpoints, endpoint)
 		}
 	}
 
 	// Check if model has preferred endpoints configured
 	if modelConfig, ok := c.ModelConfig[modelName]; ok && len(modelConfig.PreferredEndpoints) > 0 {
 		var preferredEndpoints []VLLMEndpoint
 		for _, endpoint := range availableEndpoints {
-			for _, preferredName := range modelConfig.PreferredEndpoints {
-				if endpoint.Name == preferredName {
-					preferredEndpoints = append(preferredEndpoints, endpoint)
-					break
-				}
+			if slices.Contains(modelConfig.PreferredEndpoints, endpoint.Name) {
+				preferredEndpoints = append(preferredEndpoints, endpoint)
 			}
 		}
 		if len(preferredEndpoints) > 0 {

src/semantic-router/pkg/config/config_test.go

@@ -60,7 +60,6 @@ classifier:
     use_cpu: true
     use_modernbert: false
     pii_mapping_path: "/path/to/pii.json"
-  load_aware: true
 
 categories:
   - name: "general"
@@ -138,7 +137,6 @@ tools:
 				// Verify classifier config
 				Expect(cfg.Classifier.CategoryModel.ModelID).To(Equal("test-category-model"))
 				Expect(cfg.Classifier.CategoryModel.UseModernBERT).To(BeTrue())
-				Expect(cfg.Classifier.LoadAware).To(BeTrue())
 
 				// Verify categories
 				Expect(cfg.Categories).To(HaveLen(1))

src/semantic-router/pkg/extproc/request_handler.go

@@ -370,9 +370,6 @@ func (r *OpenAIRouter) handleModelRouting(openAIRequest *openai.ChatCompletionNe
 				effortForMetrics := r.getReasoningEffort(categoryName)
 				metrics.RecordReasoningDecision(categoryName, matchedModel, useReasoning, effortForMetrics)
 
-				// Track the model load for the selected model
-				r.Classifier.IncrementModelLoad(matchedModel)
-
 				// Track the model routing change
 				metrics.RecordModelRouting(originalModel, matchedModel)
 

src/semantic-router/pkg/extproc/response_handler.go

@@ -52,7 +52,6 @@ func (r *OpenAIRouter) handleResponseBody(v *ext_proc.ProcessingRequest_Response
 			float64(completionTokens),
 		)
 		metrics.RecordModelCompletionLatency(ctx.RequestModel, completionLatency.Seconds())
-		r.Classifier.DecrementModelLoad(ctx.RequestModel)
 
 		// Compute and record cost if pricing is configured
 		if r.Config != nil {

src/semantic-router/pkg/extproc/router.go

@@ -131,8 +131,7 @@ func NewOpenAIRouter(configPath string) (*OpenAIRouter, error) {
 
 	// Create utility components
 	piiChecker := pii.NewPolicyChecker(cfg, cfg.ModelConfig)
-	modelTTFT := make(map[string]float64) // Empty TTFT map since load balancing is disabled
-	classifier := classification.NewClassifier(cfg, categoryMapping, piiMapping, jailbreakMapping, modelTTFT)
+	classifier := classification.NewClassifier(cfg, categoryMapping, piiMapping, jailbreakMapping)
 
 	// Create global classification service for API access
 	services.NewClassificationService(classifier, cfg)

src/semantic-router/pkg/extproc/security_test.go

@@ -52,7 +52,7 @@ var _ = Describe("Security Checks", func() {
 				},
 			}
 			router.PIIChecker = pii.NewPolicyChecker(cfg, cfg.ModelConfig)
-			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, router.Classifier.PIIMapping, nil, router.Classifier.ModelTTFT)
+			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, router.Classifier.PIIMapping, nil)
 		})
 
 		It("should allow requests with no PII", func() {
@@ -97,7 +97,7 @@ var _ = Describe("Security Checks", func() {
 			piiMapping, err := classification.LoadPIIMapping(cfg.Classifier.PIIModel.PIIMappingPath)
 			Expect(err).NotTo(HaveOccurred())
 
-			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, piiMapping, nil, router.Classifier.ModelTTFT)
+			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, piiMapping, nil)
 		})
 
 		Describe("ClassifyPII method", func() {
@@ -339,7 +339,7 @@ var _ = Describe("Security Checks", func() {
 			piiMapping, err := classification.LoadPIIMapping(cfg.Classifier.PIIModel.PIIMappingPath)
 			Expect(err).NotTo(HaveOccurred())
 
-			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, piiMapping, nil, router.Classifier.ModelTTFT)
+			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, piiMapping, nil)
 		})
 
 		Describe("Error handling and edge cases", func() {
@@ -524,7 +524,7 @@ var _ = Describe("Security Checks", func() {
 				IdxToLabel: map[string]string{"0": "benign", "1": "jailbreak"},
 			}
 
-			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, router.Classifier.PIIMapping, jailbreakMapping, router.Classifier.ModelTTFT)
+			router.Classifier = classification.NewClassifier(cfg, router.Classifier.CategoryMapping, router.Classifier.PIIMapping, jailbreakMapping)
 		})
 
 		It("should process potential jailbreak attempts", func() {