✨ feat(milvus): add Kubernetes deployment configs and semantic cache support

Signed-off-by: samzong <[email protected]>

Author: samzong

deploy/kserve/configmap-router-config-milvus.yaml

@@ -0,0 +1,111 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: semantic-router-kserve-config-milvus
+  labels:
+    app: semantic-router
+    component: config
+data:
+  config.yaml: |
+    bert_model:
+      model_id: models/{{EMBEDDING_MODEL}}
+      threshold: 0.6
+      use_cpu: true
+
+    semantic_cache:
+      enabled: true
+      backend_type: "milvus"
+      backend_config_path: "config/semantic-cache/milvus.yaml"
+      similarity_threshold: 0.8
+      ttl_seconds: 3600
+      embedding_model: "bert"
+
+    tools:
+      enabled: false
+      top_k: 3
+      similarity_threshold: 0.2
+      tools_db_path: "config/tools_db.json"
+      fallback_to_empty: true
+
+    prompt_guard:
+      enabled: true
+      use_modernbert: true
+      model_id: "models/jailbreak_classifier_modernbert-base_model"
+      threshold: 0.7
+      use_cpu: true
+      jailbreak_mapping_path: "models/jailbreak_classifier_modernbert-base_model/jailbreak_type_mapping.json"
+
+    vllm_endpoints:
+      - name: "{{INFERENCESERVICE_NAME}}-endpoint"
+        address: "{{PREDICTOR_SERVICE_IP}}"
+        port: 8080
+        weight: 1
+
+    model_config:
+      "{{MODEL_NAME}}":
+        reasoning_family: "qwen3"
+        preferred_endpoints: ["{{INFERENCESERVICE_NAME}}-endpoint"]
+
+    classifier:
+      category_model:
+        model_id: "models/category_classifier_modernbert-base_model"
+        use_modernbert: true
+        threshold: 0.6
+        use_cpu: true
+        category_mapping_path: "models/category_classifier_modernbert-base_model/category_mapping.json"
+      pii_model:
+        model_id: "models/pii_classifier_modernbert-base_presidio_token_model"
+        use_modernbert: true
+        threshold: 0.7
+        use_cpu: true
+        pii_mapping_path: "models/pii_classifier_modernbert-base_presidio_token_model/pii_type_mapping.json"
+
+    categories:
+      - name: business
+      - name: law
+      - name: psychology
+      - name: biology
+      - name: chemistry
+      - name: history
+      - name: other
+      - name: health
+      - name: economics
+      - name: math
+      - name: physics
+      - name: computer_science
+      - name: philosophy
+      - name: engineering
+
+    strategy: "priority"
+
+    decisions:
+      - name: "general_decision"
+        description: "General knowledge and miscellaneous topics"
+        priority: 50
+        rules:
+          operator: "AND"
+          conditions:
+            - type: "domain"
+              name: "other"
+        modelRefs:
+          - model: "{{MODEL_NAME}}"
+            use_reasoning: false
+        plugins:
+          - type: "semantic-cache"
+            configuration:
+              enabled: true
+              similarity_threshold: 0.75
+          - type: "pii"
+            configuration:
+              enabled: true
+              pii_types_allowed: []
+
+    default_model: {{MODEL_NAME}}
+
+    reasoning_families:
+      deepseek:
+        type: "chat_template_kwargs"
+        parameter: "thinking"
+      qwen3:
+        type: "chat_template_kwargs"
+        parameter: "enable_thinking"

deploy/kubernetes/milvus/config.yaml

@@ -0,0 +1,12 @@
+bert_model:
+  model_id: models/all-MiniLM-L12-v2
+  threshold: 0.6
+  use_cpu: true
+
+semantic_cache:
+  enabled: true
+  backend_type: "memory"
+  similarity_threshold: 0.8
+  ttl_seconds: 3600
+  embedding_model: "bert"
+  backend_config_path: "config/semantic-cache/milvus.yaml"

deploy/kubernetes/milvus/configmap-milvus-client.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: milvus-client-config
+  namespace: vllm-semantic-router-system
+data:
+  milvus.yaml: |
+    connection:
+      host: "milvus-cluster.vllm-semantic-router-system.svc.cluster.local"
+      port: 19530
+      timeout: 60
+      auth:
+        enabled: false
+        username: ""
+        password: ""
+      tls:
+        enabled: false
+    collection:
+      name: "semantic_cache"
+      description: "Semantic cache"
+      vector_field:
+        name: "embedding"
+        dimension: 384
+        metric_type: "IP"
+      index:
+        type: "HNSW"
+        params:
+          M: 16
+          efConstruction: 64
+    search:
+      params:
+        ef: 64
+      topk: 10
+      consistency_level: "Session"
+    development:
+      auto_create_collection: true
+      verbose_errors: true

deploy/kubernetes/milvus/milvus-cluster.yaml

@@ -0,0 +1,26 @@
+apiVersion: milvus.io/v1beta1
+kind: Milvus
+metadata:
+  name: milvus-cluster
+  namespace: vllm-semantic-router-system
+spec:
+  mode: cluster
+  components:
+    disableMetrics: false
+  dependencies:
+    storage:
+      inCluster:
+        values:
+          mode: distributed
+        deletionPolicy: Retain
+        pvcDeletion: false
+    etcd:
+      inCluster:
+        values:
+          replicaCount: 3
+    pulsar:
+      inCluster:
+        values:
+          broker:
+            replicaCount: 1
+  config: {}

deploy/kubernetes/milvus/milvus-standalone.yaml

@@ -0,0 +1,21 @@
+apiVersion: milvus.io/v1beta1
+kind: Milvus
+metadata:
+  name: milvus-standalone
+  namespace: vllm-semantic-router-system
+spec:
+  mode: standalone
+  components:
+    disableMetrics: false
+  dependencies:
+    storage:
+      inCluster:
+        values:
+          mode: standalone
+        deletionPolicy: Delete
+        pvcDeletion: true
+    etcd:
+      inCluster:
+        values:
+          replicaCount: 1
+  config: {}

deploy/kubernetes/milvus/networkpolicy.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-router-to-milvus
+  namespace: vllm-semantic-router-system
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: milvus
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: vllm-semantic-router-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: semantic-router
+      ports:
+        - protocol: TCP
+          port: 19530

deploy/kubernetes/milvus/pvc-milvus.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: milvus-data
+  namespace: vllm-semantic-router-system
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi

deploy/kubernetes/milvus/secret-milvus-auth.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: milvus-auth
+  namespace: vllm-semantic-router-system
+type: Opaque
+stringData:
+  username: ""
+  password: ""

deploy/kubernetes/milvus/servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: milvus-servicemonitor
+  namespace: vllm-semantic-router-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: milvus
+  namespaceSelector:
+    matchNames:
+      - vllm-semantic-router-system
+  endpoints:
+    - targetPort: 9091
+      path: /metrics
+      interval: 30s

deploy/kubernetes/milvus/values-cluster.yaml

@@ -0,0 +1,11 @@
+cluster:
+  enabled: true
+etcd:
+  replicaCount: 3
+minio:
+  mode: distributed
+pulsar:
+  enabled: true
+metrics:
+  serviceMonitor:
+    enabled: true