We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent a5f845a commit b2065a2Copy full SHA for b2065a2
vllm/v1/worker/hpu_model_runner.py
@@ -2538,12 +2538,8 @@ def copy_kv_blocks(
2538
src_device=src_device,
2539
dst_device=dst_device,
2540
)
2541
-
2542
- for idx, (layer, kv_layer) in enumerate(src_kv_caches):
2543
- if direction == "h2d":
2544
- k, v = kv_layer[0], kv_layer[1]
2545
- else:
2546
- k, v = kv_layer
+ for layer, kv_layer in src_kv_caches.items():
+ k, v = kv_layer[0], kv_layer[1]
2547
dst_kv_caches[layer][0][dst_indices].copy_(k[src_indices], non_blocking = False)
2548
dst_kv_caches[layer][1][dst_indices].copy_(v[src_indices], non_blocking = False)
2549
0 commit comments