File tree Expand file tree Collapse file tree 3 files changed +9
-8
lines changed
Expand file tree Collapse file tree 3 files changed +9
-8
lines changed Original file line number Diff line number Diff line change @@ -29,12 +29,12 @@ describe("extract-text", () => {
2929 } )
3030
3131 it ( "throws ContentTooLargeError when file would exceed max allowed size" , async ( ) => {
32- // Create content that would exceed max allowed size for deepseek (64k - 27k tokens)
33- const largeContent = "x" . repeat ( 148000 ) // 37k tokens > (64k - 27k) tokens
32+ // Create content that would exceed max allowed size (37k tokens)
33+ const largeContent = "x" . repeat ( 148000 ) // 37k tokens
3434 await fs . writeFile ( tempFilePath , largeContent )
3535
3636 try {
37- await extractTextFromFile ( tempFilePath , 64_000 )
37+ await extractTextFromFile ( tempFilePath , 37_000 ) // Pass pre-processed maxAllowedSize
3838 throw new Error ( "Should have thrown error" )
3939 } catch ( error ) {
4040 expect ( error ) . to . be . instanceOf ( ContentTooLargeError )
Original file line number Diff line number Diff line change @@ -16,9 +16,11 @@ describe("content-size", () => {
1616
1717 describe ( "wouldExceedSizeLimit" , ( ) => {
1818 it ( "checks if byte count would exceed max allowed size" , ( ) => {
19- expect ( wouldExceedSizeLimit ( 100 , 64_000 ) ) . to . equal ( false ) // 25 tokens < (64k - 27k) tokens
20- expect ( wouldExceedSizeLimit ( 148000 , 64_000 ) ) . to . equal ( true ) // 37k tokens > (64k - 27k) tokens
21- expect ( wouldExceedSizeLimit ( 392000 , 128_000 ) ) . to . equal ( true ) // 98k tokens > (128k - 30k) tokens
19+ // For deepseek (64k - 27k = 37k tokens)
20+ expect ( wouldExceedSizeLimit ( 100 , 37_000 ) ) . to . equal ( false ) // 25 tokens < 37k tokens
21+ expect ( wouldExceedSizeLimit ( 148000 , 37_000 ) ) . to . equal ( true ) // 37k tokens = 37k tokens
22+ // For standard models (128k - 30k = 98k tokens)
23+ expect ( wouldExceedSizeLimit ( 392000 , 98_000 ) ) . to . equal ( true ) // 98k tokens = 98k tokens
2224 } )
2325 } )
2426
Original file line number Diff line number Diff line change @@ -23,8 +23,7 @@ export function estimateTokens(bytes: number): number {
2323 */
2424export function wouldExceedSizeLimit ( byteCount : number , contextLimit : number ) : boolean {
2525 const estimatedTokenCount = estimateTokens ( byteCount )
26- const maxAllowedSize = getMaxAllowedSize ( contextLimit )
27- return estimatedTokenCount >= maxAllowedSize
26+ return estimatedTokenCount >= getMaxAllowedSize ( contextLimit )
2827}
2928
3029/**
You can’t perform that action at this time.
0 commit comments