We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 5e49421 commit 98c8cc9Copy full SHA for 98c8cc9
src/core/sliding-window/index.ts
@@ -89,9 +89,9 @@ function getMaxTokensForNonPromptCachingModels(modelInfo: ModelInfo): number {
89
/**
90
* Provides the fraction of messages to remove for models that do not support prompt caching.
91
*
92
- * @param {ModelInfo} modelInfo - The model information (unused in current implementation).
+ * @param {ModelInfo} modelInfo - The model information.
93
* @returns {number} The truncation fraction for non-prompt caching models (fixed at 0.1).
94
*/
95
function getTruncFractionForNonPromptCachingModels(modelInfo: ModelInfo): number {
96
- return 0.1
+ return Math.min(40_000 / modelInfo.contextWindow, 0.2)
97
}
0 commit comments