[Feat]: VSR + public LLM/ OpenAI + local llm + istio + LLM-d deployment guide (#643)

Signed-off-by: Sanjeev Rampal <[email protected]>

Author: srampal

deploy/kubernetes/istio/README.md

@@ -1,7 +1,11 @@
-# vLLM Semantic Router as ExtProc server for Istio Gateway 
+# vLLM Semantic Router as ExtProc server for Istio Gateway
+
+This guide provides step-by-step instructions for deploying the vLLM Semantic Router (vSR) with Istio Gateway on Kubernetes. Istio Gateway uses Envoy under the covers so it is possible to use vSR with it. Istio is a common choice for the gateway when using Kubernetes Gateway API Inference Extension and in the LLM-D project as well as in common Kubernetes distributions such as Red Hat Openshift. In our experience, there are low level differences in how different Envoy based gateways process the ExtProc protocol to assist with LLM inference, hence this guide and some others cover the specific case of vSR working with an Istio based gateway.
+
+There are multiple deployment guides in this repo related to vSR+Istio deployments. This current document describes deployment of vSR with Istio gateway and two local LLMs served using vLLM. Additional deployment guides in this repo build on this deployment to add support for integrating LLM-D and to illustrate support for routing to remote/ public cloud LLMs. Those topics are covered by other followup deployment guides in this repo ([llm-d guide](../llmd-base/README.md) and [public llm routing guide](../llmd-base/llmd+public-llm/README.md). 
+
+With that background context in mind, we now follow this guide to describe the vSR + Istio + locally hosted LLMs use case. After this guide, the reader may then optionally choose to follow up with the additional guides linked above to deploy the more advanced use cases.
 
-This guide provides step-by-step instructions for deploying the vLLM Semantic Router (vsr) with Istio Gateway on Kubernetes. Istio Gateway uses Envoy under the covers so it is possible to use vsr with it. However there are differences between how different Envoy based Gateways process the ExtProc protocol, hence the deployment described here is different from the deployment of vsr alongwith other types of Envoy based Gateways as described in the other guides in this repo. There are multiple architecture options possible to combine Istio Gateway with vsr. This document describes one of the options.
- 
 ## Architecture Overview
 
 The deployment consists of:

deploy/kubernetes/llmd-base/llmd+public-llm/README.md

@@ -0,0 +1,240 @@
+bert_model:
+  model_id: models/all-MiniLM-L12-v2
+  threshold: 0.6
+  use_cpu: true
+
+semantic_cache:
+  enabled: false
+  backend_type: "memory"  # Options: "memory" or "milvus"
+  similarity_threshold: 0.8
+  max_entries: 1000  # Only applies to memory backend
+  ttl_seconds: 3600
+  eviction_policy: "fifo"
+  # Embedding model for semantic similarity matching
+  # Options: "bert" (fast, 384-dim), "qwen3" (high quality, 1024-dim, 32K context), "gemma" (balanced, 768-dim, 8K context)
+  embedding_model: "bert"  # Default: BERT (fastest, lowest memory for Kubernetes)
+
+tools:
+  enabled: false
+  top_k: 3
+  similarity_threshold: 0.2
+  tools_db_path: "config/tools_db.json"
+  fallback_to_empty: true
+
+prompt_guard:
+  enabled: false  # Global default - can be overridden per category with jailbreak_enabled
+  use_modernbert: true
+  model_id: "models/jailbreak_classifier_modernbert-base_model"
+  threshold: 0.7
+  use_cpu: true
+  jailbreak_mapping_path: "models/jailbreak_classifier_modernbert-base_model/jailbreak_type_mapping.json"
+
+# vLLM Endpoints Configuration
+# IMPORTANT: 'address' field must be a valid IP address (IPv4 or IPv6)
+# Supported formats: 127.0.0.1, 192.168.1.1, ::1, 2001:db8::1
+# NOT supported: domain names (example.com), protocol prefixes (http://), paths (/api), ports in address (use 'port' field)
+vllm_endpoints:
+  - name: "endpoint1"
+    address: "10.98.150.102"  # Static IPv4 of llama3-8b k8s service
+    port: 80
+    weight: 1
+  - name: "endpoint2"
+    address: "10.98.118.242"  # Static IPv4 of phi4-mini k8s service
+    port: 80
+    weight: 1
+
+model_config:
+  "llama3-8b":
+    #    reasoning_family: "" # This model uses Qwen-3 reasoning syntax
+    preferred_endpoints: ["endpoint1"]
+    pii_policy:
+      allow_by_default: true
+  "phi4-mini":
+    #    reasoning_family: "" # This model uses Qwen-3 reasoning syntax
+    preferred_endpoints: ["endpoint2"]
+    pii_policy:
+      allow_by_default: true
+
+# Classifier configuration
+classifier:
+  category_model:
+    model_id: "models/category_classifier_modernbert-base_model"
+    use_modernbert: true
+    threshold: 0.6
+    use_cpu: true
+    category_mapping_path: "models/category_classifier_modernbert-base_model/category_mapping.json"
+  pii_model:
+    model_id: "models/pii_classifier_modernbert-base_presidio_token_model"
+    use_modernbert: true
+    threshold: 0.7
+    use_cpu: true
+    pii_mapping_path: "models/pii_classifier_modernbert-base_presidio_token_model/pii_type_mapping.json"
+
+# Categories with new use_reasoning field structure
+categories:
+  - name: business
+    system_prompt: "You are a senior business consultant and strategic advisor with expertise in corporate strategy, operations management, financial analysis, marketing, and organizational development. Provide practical, actionable business advice backed by proven methodologies and industry best practices. Consider market dynamics, competitive landscape, and stakeholder interests in your recommendations."
+    # jailbreak_enabled: true  # Optional: Override global jailbreak detection per category
+    # jailbreak_threshold: 0.8  # Optional: Override global jailbreak threshold per category
+    model_scores:
+      - model: llama3-8b
+        score: 0.8
+        use_reasoning: false  # Business performs better without reasoning
+      - model: phi4-mini
+        score: 0.3
+        use_reasoning: false  # Business performs better without reasoning
+  - name: law
+    system_prompt: "You are a knowledgeable legal expert with comprehensive understanding of legal principles, case law, statutory interpretation, and legal procedures across multiple jurisdictions. Provide accurate legal information and analysis while clearly stating that your responses are for informational purposes only and do not constitute legal advice. Always recommend consulting with qualified legal professionals for specific legal matters."
+    model_scores:
+      - model: llama3-8b
+        score: 0.4
+        use_reasoning: false
+  - name: psychology
+    system_prompt: "You are a psychology expert with deep knowledge of cognitive processes, behavioral patterns, mental health, developmental psychology, social psychology, and therapeutic approaches. Provide evidence-based insights grounded in psychological research and theory. When discussing mental health topics, emphasize the importance of professional consultation and avoid providing diagnostic or therapeutic advice."
+    semantic_cache_enabled: true
+    semantic_cache_similarity_threshold: 0.92  # High threshold for psychology - sensitive to nuances
+    model_scores:
+      - model: llama3-8b
+        score: 0.6
+        use_reasoning: false
+  - name: biology
+    system_prompt: "You are a biology expert with comprehensive knowledge spanning molecular biology, genetics, cell biology, ecology, evolution, anatomy, physiology, and biotechnology. Explain biological concepts with scientific accuracy, use appropriate terminology, and provide examples from current research. Connect biological principles to real-world applications and emphasize the interconnectedness of biological systems."
+    model_scores:
+      - model: llama3-8b
+        score: 0.9
+        use_reasoning: false
+  - name: chemistry
+    system_prompt: "You are a chemistry expert specializing in chemical reactions, molecular structures, and laboratory techniques. Provide detailed, step-by-step explanations."
+    model_scores:
+      - model: llama3-8b
+        score: 0.6
+        use_reasoning: false  # Enable reasoning for complex chemistry
+  - name: history
+    system_prompt: "You are a historian with expertise across different time periods and cultures. Provide accurate historical context and analysis."
+    model_scores:
+      - model: llama3-8b
+        score: 0.7
+        use_reasoning: false
+  - name: other
+    system_prompt: "You are a helpful and knowledgeable assistant. Provide accurate, helpful responses across a wide range of topics."
+    semantic_cache_enabled: true
+    semantic_cache_similarity_threshold: 0.75  # Lower threshold for general chat - less sensitive
+    model_scores:
+      - model: llama3-8b
+        score: 0.7
+        use_reasoning: false
+  - name: health
+    system_prompt: "You are a health and medical information expert with knowledge of anatomy, physiology, diseases, treatments, preventive care, nutrition, and wellness. Provide accurate, evidence-based health information while emphasizing that your responses are for educational purposes only and should never replace professional medical advice, diagnosis, or treatment. Always encourage users to consult healthcare professionals for medical concerns and emergencies."
+    semantic_cache_enabled: true
+    semantic_cache_similarity_threshold: 0.95  # High threshold for health - very sensitive to word changes
+    model_scores:
+      - model: llama3-8b
+        score: 0.5
+        use_reasoning: false
+  - name: economics
+    system_prompt: "You are an economics expert with deep understanding of microeconomics, macroeconomics, econometrics, financial markets, monetary policy, fiscal policy, international trade, and economic theory. Analyze economic phenomena using established economic principles, provide data-driven insights, and explain complex economic concepts in accessible terms. Consider both theoretical frameworks and real-world applications in your responses."
+    model_scores:
+      - model: llama3-8b
+        score: 1.0
+        use_reasoning: false
+  - name: math
+    system_prompt: "You are a mathematics expert. Provide step-by-step solutions, show your work clearly, and explain mathematical concepts in an understandable way."
+    model_scores:
+      - model: phi4-mini
+        score: 1.0
+        use_reasoning: false  # Enable reasoning for complex math
+  - name: physics
+    system_prompt: "You are a physics expert with deep understanding of physical laws and phenomena. Provide clear explanations with mathematical derivations when appropriate."
+    model_scores:
+      - model: llama3-8b
+        score: 0.7
+        use_reasoning: false  # Enable reasoning for physics
+  - name: computer science
+    system_prompt: "You are a computer science expert with knowledge of algorithms, data structures, programming languages, and software engineering. Provide clear, practical solutions with code examples when helpful."
+    model_scores:
+      - model: llama3-8b
+        score: 0.6
+        use_reasoning: false
+  - name: philosophy
+    system_prompt: "You are a philosophy expert with comprehensive knowledge of philosophical traditions, ethical theories, logic, metaphysics, epistemology, political philosophy, and the history of philosophical thought. Engage with complex philosophical questions by presenting multiple perspectives, analyzing arguments rigorously, and encouraging critical thinking. Draw connections between philosophical concepts and contemporary issues while maintaining intellectual honesty about the complexity and ongoing nature of philosophical debates."
+    model_scores:
+      - model: llama3-8b
+        score: 0.5
+        use_reasoning: false
+  - name: engineering
+    system_prompt: "You are an engineering expert with knowledge across multiple engineering disciplines including mechanical, electrical, civil, chemical, software, and systems engineering. Apply engineering principles, design methodologies, and problem-solving approaches to provide practical solutions. Consider safety, efficiency, sustainability, and cost-effectiveness in your recommendations. Use technical precision while explaining concepts clearly, and emphasize the importance of proper engineering practices and standards."
+    model_scores:
+      - model: llama3-8b
+        score: 0.7
+        use_reasoning: false
+
+default_model: "llama3-8b"
+
+# Auto model name for automatic model selection (optional)
+# This is the model name that clients should use to trigger automatic model selection
+# If not specified, defaults to "MoM" (Mixture of Models)
+# For backward compatibility, "auto" is always accepted as an alias
+# Example: auto_model_name: "MoM"  # or any other name you prefer
+# auto_model_name: "MoM"
+
+# Include configured models in /v1/models list endpoint (optional, default: false)
+# When false (default): only the auto model name is returned in the /v1/models endpoint
+# When true: all models configured in model_config are also included in the /v1/models endpoint
+# This is useful for clients that need to discover all available models
+# Example: include_config_models_in_list: true
+# include_config_models_in_list: false
+
+# Reasoning family configurations
+reasoning_families:
+  deepseek:
+    type: "chat_template_kwargs"
+    parameter: "thinking"
+
+  qwen3:
+    type: "chat_template_kwargs"
+    parameter: "enable_thinking"
+
+  gpt-oss:
+    type: "reasoning_effort"
+    parameter: "reasoning_effort"
+  gpt:
+    type: "reasoning_effort"
+    parameter: "reasoning_effort"
+
+# Global default reasoning effort level
+default_reasoning_effort: high
+
+# Gateway route cache clearing
+clear_route_cache: true  # Enable for some gateways such as Istio
+
+# API Configuration
+api:
+  batch_classification:
+    max_batch_size: 100
+    concurrency_threshold: 5
+    max_concurrency: 8
+    metrics:
+      enabled: true
+      detailed_goroutine_tracking: true
+      high_resolution_timing: false
+      sample_rate: 1.0
+      duration_buckets:
+        [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]
+      size_buckets: [1, 2, 5, 10, 20, 50, 100, 200]
+
+# Observability Configuration
+observability:
+  tracing:
+    enabled: true  # Enable distributed tracing for docker-compose stack
+    provider: "opentelemetry"  # Provider: opentelemetry, openinference, openllmetry
+    exporter:
+      type: "otlp"  # Export spans to Jaeger (via OTLP gRPC)
+      endpoint: "jaeger:4317"  # Jaeger collector inside compose network
+      insecure: true  # Use insecure connection (no TLS)
+    sampling:
+      type: "always_on"  # Sampling: always_on, always_off, probabilistic
+      rate: 1.0  # Sampling rate for probabilistic (0.0-1.0)
+    resource:
+      service_name: "vllm-semantic-router"
+      service_version: "v0.1.0"
+      deployment_environment: "development"