Skip to content

Commit b7e893f

Browse files
committed
Fix disagg with kvcache reuse for dsv32
Signed-off-by: Iman Tabrizian <10105175+tabrizian@users.noreply.github.com>
1 parent 33545f8 commit b7e893f

File tree

2 files changed

+5
-1
lines changed

2 files changed

+5
-1
lines changed

cpp/include/tensorrt_llm/batch_manager/kvCacheUtils.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,10 @@ class BlockRange
183183
auto windowSize = cacheManager.getBlockManager().getPoolWindowSize(poolIdx);
184184
mPoolsPerWindow[windowSize].push_back(cacheManager.getBlockManager().getPrimaryPool(poolIdx));
185185
}
186+
if (cacheManager.isEnableIndexerKCache())
187+
{
188+
mIndexerKCachePool = cacheManager.getIndexerKCachePool();
189+
}
186190
}
187191

188192
BlockRange(BaseKVCacheManager const& cacheManager, LlmRequest::RequestIdType requestId)

cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,7 @@ class CacheReceiver::Impl
806806

807807
RequestInfo requestInfo(requestId, mSelfState);
808808

809-
if (mFormatter->getCacheManager()->getBlockManager().getNumPools() == 1)
809+
if (!mFormatter->getCacheManager()->getBlockManager().isVariableWindow())
810810
{
811811
auto* cacheManager = mFormatter->getCacheManager();
812812
auto beam = 0;

0 commit comments

Comments
 (0)