We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 1282ebe commit 70633c6Copy full SHA for 70633c6
fastdeploy/cache_manager/cache_messager.py
@@ -152,8 +152,8 @@ def __init__(
152
cache_v = []
153
self.messager = {}
154
for layer_idx in range(self.num_layers):
155
- key_cache = self.gpu_cache_kvs[f"key_caches_{layer_idx}_rank{self.rank}.device{gpu_id}"]
156
- val_cache = self.gpu_cache_kvs[f"value_caches_{layer_idx}_rank{self.rank}.device{gpu_id}"]
+ key_cache = self.gpu_cache_kvs[f"key_caches_{layer_idx}_rank{self.rank}_device{gpu_id}"]
+ val_cache = self.gpu_cache_kvs[f"value_caches_{layer_idx}_rank{self.rank}_device{gpu_id}"]
157
cache_k.append(key_cache)
158
cache_v.append(val_cache)
159
cache_k_ptr_list.append(key_cache.data_ptr())
0 commit comments