Update Prefix-Cache-Scorer cache_tracking Mode with The v0.2 KVCache.Indexer (#245)

* Support for new version of kv-cache-manager is added to scheduler

* Redundant function parameter and struct member are removed

* Minor changes in comments

* Redundant setting of ZMQEndpoint is removed

* - general refactoring
- enhanced prefix-cache-scorer cache-tracking mode's configurability
- updated docs
- enhanced docs (no HTML use)

Signed-off-by: Maroon Ayoub <[email protected]>

* refactor example configs

Signed-off-by: Maroon Ayoub <[email protected]>

---------

Signed-off-by: Maroon Ayoub <[email protected]>
Co-authored-by: Maroon Ayoub <[email protected]>
Co-authored-by: Dmitri Pikus <[email protected]>

Author: dmitripikus

.github/workflows/ci-pr-checks.yaml

@@ -25,6 +25,14 @@ jobs:
           go-version: "${{ env.GO_VERSION }}"
           cache-dependency-path: ./go.sum
 
+      - name: Install libzmq dependencies (kvcache/kvevents)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libzmq3-dev pkg-config
+
+      - name: Set PKG_CONFIG_PATH
+        run: echo "PKG_CONFIG_PATH=/usr/lib/pkgconfig" >> $GITHUB_ENV
+
       - name: go mod tidy
         run: go mod tidy
 

Dockerfile

@@ -4,7 +4,10 @@ ARG TARGETOS
 ARG TARGETARCH
 
 # Install build tools
-RUN dnf install -y gcc-c++ libstdc++ libstdc++-devel clang && dnf clean all
+# The builder is based on UBI8, so we need epel-release-8.
+RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' && \
+    dnf install -y gcc-c++ libstdc++ libstdc++-devel clang zeromq-devel pkgconfig && \
+    dnf clean all
 
 WORKDIR /workspace
 
@@ -36,11 +39,22 @@ RUN go build -a -o bin/epp -ldflags="-extldflags '-L$(pwd)/lib'" cmd/epp/main.go
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
 WORKDIR /
 COPY --from=builder /workspace/bin/epp /app/epp
+
+# Install zeromq runtime library needed by the manager.
+# The final image is UBI9, so we need epel-release-9.
+USER root
+RUN microdnf install -y dnf && \
+    dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' && \
+    dnf install -y zeromq
+
 USER 65532:65532
 
 # expose gRPC, health and metrics ports
 EXPOSE 9002
 EXPOSE 9003
 EXPOSE 9090
 
+# expose port for KV-Events ZMQ SUB socket
+EXPOSE 5557
+
 ENTRYPOINT ["/app/epp"]

deploy/config/epp-prefix-cache-tracking-config.yaml

@@ -3,20 +3,28 @@
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: single-profile-handler
-- type: decode-filter
-- type: prefix-cache-scorer
-  parameters:
-    mode: cache_tracking
-    kvCacheRedisAddr: ${REDIS_HOST}:${REDIS_PORT}
-- type: load-aware-scorer
-- type: max-score-picker
+  - type: single-profile-handler
+  - type: decode-filter
+  - type: prefix-cache-scorer
+    parameters:
+      mode: cache_tracking
+      indexerConfig:
+        tokenProcessorConfig:
+          blockSize: 64                 # must match vLLM block size
+          hashSeed: "42"                # must match vLLM PYTHONHASHSEED env var
+        kvBlockIndexConfig:
+          enableMetrics: true           # enable kv-block index metrics (prometheus)
+  - type: kv-cache-scorer # kv-cache-utilization
+  - type: queue-scorer
+  - type: max-score-picker
 schedulingProfiles:
-- name: default
-  plugins:
-  - pluginRef: decode-filter
-  - pluginRef: prefix-cache-scorer
-    weight: 2.0
-  - pluginRef: load-aware-scorer
-    weight: 1.0
-  - pluginRef: max-score-picker
+  - name: default
+    plugins:
+      - pluginRef: decode-filter
+      - pluginRef: prefix-cache-scorer
+        weight: 3.0
+      - pluginRef: kv-cache-scorer
+        weight: 1.0
+      - pluginRef: queue-scorer
+        weight: 1.0
+      - pluginRef: max-score-picker