Matched Default TokenProcessorConfig.BlockSize with vLLM's (#52)

* - update vLLM deployment in chart
- update default tokens hashing block-size

Signed-off-by: Maroon Ayoub <[email protected]>

* updated default kvevents::Config::ZMQEndpoint

Signed-off-by: Maroon Ayoub <[email protected]>

* clarify defaultBlockSize

Signed-off-by: Maroon Ayoub <[email protected]>

---------

Signed-off-by: Maroon Ayoub <[email protected]>

Author: vMaroon

examples/kv_events/online/main.go

@@ -66,7 +66,7 @@ func getKVCacheIndexerConfig() *kvcache.Config {
 
 	blockSize, err := strconv.Atoi(os.Getenv(blockSizeEnvVar))
 	if err == nil || blockSize >= 0 {
-		config.TokenProcessorConfig.ChunkSize = blockSize
+		config.TokenProcessorConfig.BlockSize = blockSize
 	}
 
 	return config

pkg/kvcache/kvblock/token_processor.go

@@ -27,11 +27,13 @@ import (
 	"github.com/llm-d/llm-d-kv-cache-manager/pkg/utils"
 )
 
-const defaultChunkSize = 256
+// defaultBlockSize is the default number of tokens per block.
+// 16 is the default value used by vLLM.
+const defaultBlockSize = 16
 
 // TokenProcessorConfig holds the configuration for the token processor.
 type TokenProcessorConfig struct {
-	ChunkSize int
+	BlockSize int
 	// HashSeed is used to prefix initial hash chunks, similarly to vLLM's NONE_HASH.
 	// This should be aligned with vLLM's `PYTHONHASHSEED` environment variable.
 	// The system's deployer is responsible for aligning the vLLM deployments
@@ -44,7 +46,7 @@ type TokenProcessorConfig struct {
 // DefaultTokenProcessorConfig returns the default configuration for the token processor.
 func DefaultTokenProcessorConfig() *TokenProcessorConfig {
 	return &TokenProcessorConfig{
-		ChunkSize: defaultChunkSize,
+		BlockSize: defaultBlockSize,
 		HashSeed:  "",
 	}
 }
@@ -134,8 +136,8 @@ func (db *ChunkedTokenDatabase) prefixHashes(parentHash uint64, tokenChunks [][]
 // chunkTokens splits the input slice of tokens into chunks of size chunkSize.
 func (db *ChunkedTokenDatabase) chunkTokens(tokens []uint32) [][]uint32 {
 	var chunks [][]uint32
-	for i := 0; i < len(tokens); i += db.ChunkSize {
-		end := i + db.ChunkSize
+	for i := 0; i < len(tokens); i += db.BlockSize {
+		end := i + db.BlockSize
 		if end > len(tokens) {
 			break // no partial blocks
 		}

pkg/kvcache/kvevents/pool.go

@@ -27,7 +27,7 @@ type Config struct {
 // DefaultConfig returns a default configuration for the event processing pool.
 func DefaultConfig() *Config {
 	return &Config{
-		ZMQEndpoint: "tcp://0.0.0.0:5557",
+		ZMQEndpoint: "tcp://*:5557",
 		TopicFilter: "kv@",
 		Concurrency: 4,
 	}

tests/e2e/redis_mock/e2e_suite_test.go

@@ -60,7 +60,7 @@ func (s *KVCacheSuite) SetupTest() {
 
 	s.config = kvcache.NewDefaultConfig()
 	s.config.PrefixStoreConfig.BlockSize = 4
-	s.config.TokenProcessorConfig.ChunkSize = 4
+	s.config.TokenProcessorConfig.BlockSize = 4
 
 	s.tokenizer, err = tokenization.NewCachedHFTokenizer(s.config.TokenizersPoolConfig.HFTokenizerConfig)
 	s.Require().NoError(err)

vllm-setup-helm/templates/deployment.yaml

@@ -52,14 +52,16 @@ spec:
           args:
             - |
               git config --global --add safe.directory /workspace/vllm && \
-              git remote add maroon https://github.com/vMaroon/vllm.git && \
-              git fetch maroon && \
-              git reset --hard 786715333f822ff9274700d343d8628ef29ec525 && \
+              git remote add vllm https://github.com/vllm-project/vllm.git && \
+              git fetch vllm && \
+              git reset --hard 697ef765ee91d1a47b49ae7e43951cfd116b6052 && \
               VLLM_COMMIT=$(git merge-base HEAD origin/main) && \
               VLLM_PRECOMPILED_WHEEL_LOCATION=https://wheels.vllm.ai/${VLLM_COMMIT}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl && \
               VLLM_USE_PRECOMPILED=1 uv pip install --editable . && \
               pip install -r requirements/common.txt && \
+              {{- if .Values.lmcache.enabled }}
               export LMCACHE_DISTRIBUTED_URL=${POD_IP} && \
+              {{- end }}
               vllm serve {{ .Values.vllm.model.name }} \
               --host 0.0.0.0 \
               --port 8000 \
@@ -83,7 +85,7 @@ spec:
               --block-size {{ .Values.vllm.blockSize }} \
               {{- if .Values.kvCacheManager.enabled }}
               --kv-events-config "{\"enable_kv_cache_events\":{{ .Values.kvCacheManager.enabled }},\"publisher\":\"zmq\",\"endpoint\":\"{{ include "chart.kvCacheManagerServiceUrl" . }}\",\"topic\":\"kv@${POD_IP}@{{ .Values.vllm.model.name }}\"}" \
-              --prefix-caching-hash-algo sha256_cbor \
+              --prefix-caching-hash-algo sha256_cbor_64bit \
               {{- end }}
           ports:
             - name: http