Skip to content

Commit 2aaa5b3

Browse files
pierDipipierDipi
committed
Bump kv-cache-manager to v0.4.0-rc1
Signed-off-by: Pierangelo Di Pilato <[email protected]>
1 parent 5c58b12 commit 2aaa5b3

File tree

7 files changed

+42
-20
lines changed

7 files changed

+42
-20
lines changed

‎go.mod‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ require (
88
github.com/buaazp/fasthttprouter v0.1.1
99
github.com/go-logr/logr v1.4.2
1010
github.com/google/uuid v1.6.0
11-
github.com/llm-d/llm-d-kv-cache-manager v0.3.0-rc1
11+
github.com/llm-d/llm-d-kv-cache-manager v0.4.0-rc1
1212
github.com/onsi/ginkgo/v2 v2.23.4
1313
github.com/onsi/gomega v1.37.0
1414
github.com/openai/openai-go/v3 v3.6.1
@@ -23,6 +23,12 @@ require (
2323
k8s.io/klog/v2 v2.130.1
2424
)
2525

26+
require (
27+
github.com/dgraph-io/ristretto/v2 v2.3.0 // indirect
28+
github.com/dustin/go-humanize v1.0.1 // indirect
29+
go.uber.org/multierr v1.11.0 // indirect
30+
)
31+
2632
require (
2733
github.com/andybalholm/brotli v1.1.1 // indirect
2834
github.com/beorn7/perks v1.0.1 // indirect
@@ -64,7 +70,7 @@ require (
6470
go.uber.org/automaxprocs v1.6.0 // indirect
6571
golang.org/x/net v0.38.0 // indirect
6672
golang.org/x/oauth2 v0.27.0 // indirect
67-
golang.org/x/sys v0.32.0 // indirect
73+
golang.org/x/sys v0.35.0 // indirect
6874
golang.org/x/term v0.30.0 // indirect
6975
golang.org/x/text v0.23.0 // indirect
7076
golang.org/x/time v0.9.0 // indirect
@@ -77,7 +83,7 @@ require (
7783
k8s.io/client-go v0.33.0 // indirect
7884
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
7985
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
80-
sigs.k8s.io/controller-runtime v0.21.0 // indirect
86+
sigs.k8s.io/controller-runtime v0.21.0
8187
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
8288
sigs.k8s.io/randfill v1.0.0 // indirect
8389
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect

‎go.sum‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
1919
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2020
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
2121
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
22+
github.com/dgraph-io/ristretto/v2 v2.3.0 h1:qTQ38m7oIyd4GAed/QkUZyPFNMnvVWyazGXRwvOt5zk=
23+
github.com/dgraph-io/ristretto/v2 v2.3.0/go.mod h1:gpoRV3VzrEY1a9dWAYV6T1U7YzfgttXdd/ZzL1s9OZM=
24+
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
25+
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
2226
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
2327
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
28+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
29+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
2430
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
2531
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
2632
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
@@ -68,8 +74,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
6874
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
6975
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
7076
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
71-
github.com/llm-d/llm-d-kv-cache-manager v0.3.0-rc1 h1:SDLiNrcreDcA9m9wfXAumFARDHHXpjOjHTzshTiTGxk=
72-
github.com/llm-d/llm-d-kv-cache-manager v0.3.0-rc1/go.mod h1:tN80/D0Faf6pE2ocwFgTNoCxKPsqdsa2XnjQUqOaZ8Q=
77+
github.com/llm-d/llm-d-kv-cache-manager v0.4.0-rc1 h1:gWkZ9yp7sU5j1vbNB7eO95lxbvgJV+qd/60LnPfNk9w=
78+
github.com/llm-d/llm-d-kv-cache-manager v0.4.0-rc1/go.mod h1:oEmDhEjW1pEoOSlEFy8CKoMc7ixQmSKEbhLt9CoH/a0=
7379
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
7480
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
7581
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
@@ -153,6 +159,8 @@ go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
153159
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
154160
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
155161
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
162+
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
163+
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
156164
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
157165
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
158166
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -174,8 +182,8 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
174182
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
175183
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
176184
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
177-
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
178-
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
185+
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
186+
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
179187
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
180188
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
181189
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

‎pkg/kv-cache/block_cache.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,14 +177,14 @@ func (bc *blockCache) startRequest(requestID string, blocks []uint64) (int, erro
177177

178178
delete(bc.unusedBlocks, oldestUnusedHash)
179179
common.WriteToChannel(bc.eventChan,
180-
EventData{action: eventActionRemove, hashValues: []uint64{oldestUnusedHash}},
180+
EventData{action: eventActionRemove, hashValues: []any{oldestUnusedHash}},
181181
bc.logger, "block cache eventChan")
182182
}
183183

184184
// Add the new block
185185
bc.usedBlocks[block] = 1
186186
common.WriteToChannel(bc.eventChan,
187-
EventData{action: eventActionStore, hashValues: []uint64{block}},
187+
EventData{action: eventActionStore, hashValues: []any{block}},
188188
bc.logger, "block cache eventChan")
189189
}
190190

‎pkg/kv-cache/kv_cache_sender.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ const (
3737

3838
type EventData struct {
3939
action EventAction
40-
hashValues []uint64
40+
hashValues []any
4141
}
4242

4343
type KVEventSender struct {

‎pkg/kv-cache/kv_cache_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,8 @@ var _ = Describe("KV cache", Ordered, func() {
330330
wg.Wait() // wait for goroutine to exit
331331
}()
332332

333-
expectedRemovedBlocks := []uint64{2, 4}
334-
expectedStoredBlocks := []uint64{1, 2, 3, 4, 5, 6}
333+
expectedRemovedBlocks := []any{uint64(2), uint64(4)}
334+
expectedStoredBlocks := []any{uint64(1), uint64(2), uint64(3), uint64(4), uint64(5), uint64(6)}
335335

336336
go func() {
337337
// Make sure that the subscriber listens before the events are published
@@ -371,8 +371,8 @@ var _ = Describe("KV cache", Ordered, func() {
371371
Expect(alreadyInCache).To(Equal(0))
372372
}()
373373

374-
removedBlocks := make([]uint64, 0)
375-
storedBlocks := make([]uint64, 0)
374+
removedBlocks := make([]any, 0)
375+
storedBlocks := make([]any, 0)
376376
count := uint64(1)
377377
for {
378378
parts, err := sub.RecvMessageBytes(0)

‎pkg/kv-cache/kv_test_helper.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ import (
2121

2222
"github.com/llm-d/llm-d-kv-cache-manager/pkg/kvcache/kvevents"
2323
"github.com/onsi/ginkgo/v2"
24-
gomega "github.com/onsi/gomega"
24+
"github.com/onsi/gomega"
2525
"github.com/vmihailenco/msgpack/v5"
2626
)
2727

28-
func ParseKVEvent(parts [][]byte, expectedTopic string, expectedSeq uint64) ([]uint64, []uint64, bool) {
28+
func ParseKVEvent(parts [][]byte, expectedTopic string, expectedSeq uint64) ([]any, []any, bool) {
2929
// The message should be [topic, seq, payload]
3030
gomega.Expect(parts).To(gomega.HaveLen(3))
3131

@@ -34,8 +34,8 @@ func ParseKVEvent(parts [][]byte, expectedTopic string, expectedSeq uint64) ([]u
3434
seq := binary.BigEndian.Uint64(parts[1])
3535
gomega.Expect(seq).To(gomega.Equal(expectedSeq))
3636

37-
removed := make([]uint64, 0)
38-
stored := make([]uint64, 0)
37+
removed := make([]any, 0)
38+
stored := make([]any, 0)
3939
allCleared := false
4040

4141
var eventBatch kvevents.EventBatch

‎pkg/llm-d-inference-sim/simulator.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,10 +333,18 @@ func (s *VllmSimulator) initializeSim(ctx context.Context) error {
333333
return err
334334
}
335335

336-
tokenizationConfig := tokenization.DefaultConfig()
336+
tokenizationConfig, err := tokenization.DefaultConfig()
337+
if err != nil {
338+
return fmt.Errorf("failed to create default tokenization configuration: %w", err)
339+
}
340+
337341
if s.config.TokenizersCacheDir != "" {
338-
tokenizationConfig.TokenizersCacheDir = s.config.TokenizersCacheDir
342+
if tokenizationConfig.HFTokenizerConfig == nil {
343+
tokenizationConfig.HFTokenizerConfig = &tokenization.HFTokenizerConfig{}
344+
}
345+
tokenizationConfig.HFTokenizerConfig.TokenizersCacheDir = s.config.TokenizersCacheDir
339346
}
347+
340348
s.tokenizer, err = tokenization.NewCachedHFTokenizer(tokenizationConfig.HFTokenizerConfig)
341349
if err != nil {
342350
return fmt.Errorf("failed to create tokenizer: %w", err)

0 commit comments

Comments
 (0)