@@ -9,18 +9,18 @@ const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed siz
99describe ( "content-size" , ( ) => {
1010 describe ( "estimateTokens" , ( ) => {
1111 it ( "estimates tokens based on byte count" , ( ) => {
12- expect ( estimateTokens ( 100 ) ) . to . equal ( 25 ) // 100 bytes / 4 chars per token = 25 tokens
13- expect ( estimateTokens ( 7 ) ) . to . equal ( 2 ) // Should round up for partial tokens
12+ expect ( estimateTokens ( 100 ) ) . to . equal ( 50 ) // 100 bytes / 2 chars per token = 50 tokens
13+ expect ( estimateTokens ( 7 ) ) . to . equal ( 4 ) // Should round up for partial tokens
1414 } )
1515 } )
1616
1717 describe ( "wouldExceedSizeLimit" , ( ) => {
1818 it ( "checks if byte count would exceed max allowed size" , ( ) => {
1919 // For deepseek (64k - 27k = 37k tokens)
20- expect ( wouldExceedSizeLimit ( 100 , 37_000 ) ) . to . equal ( false ) // 25 tokens < 37k tokens
21- expect ( wouldExceedSizeLimit ( 148000 , 37_000 ) ) . to . equal ( true ) // 37k tokens = 37k tokens
20+ expect ( wouldExceedSizeLimit ( 100 , 37_000 ) ) . to . equal ( false ) // 50 tokens < 37k tokens
21+ expect ( wouldExceedSizeLimit ( 148000 , 37_000 ) ) . to . equal ( true ) // 74k tokens > 37k tokens
2222 // For standard models (128k - 30k = 98k tokens)
23- expect ( wouldExceedSizeLimit ( 392000 , 98_000 ) ) . to . equal ( true ) // 98k tokens = 98k tokens
23+ expect ( wouldExceedSizeLimit ( 392000 , 98_000 ) ) . to . equal ( true ) // 196k tokens > 98k tokens
2424 } )
2525 } )
2626
@@ -30,7 +30,7 @@ describe("content-size", () => {
3030 const result = estimateContentSize ( content , CONTEXT_LIMIT )
3131
3232 expect ( result . bytes ) . to . equal ( 11 )
33- expect ( result . estimatedTokens ) . to . equal ( 3 )
33+ expect ( result . estimatedTokens ) . to . equal ( 6 )
3434 expect ( result . wouldExceedLimit ) . to . equal ( false )
3535 } )
3636
@@ -39,13 +39,13 @@ describe("content-size", () => {
3939 const result = estimateContentSize ( content , CONTEXT_LIMIT )
4040
4141 expect ( result . bytes ) . to . equal ( 11 )
42- expect ( result . estimatedTokens ) . to . equal ( 3 )
42+ expect ( result . estimatedTokens ) . to . equal ( 6 )
4343 expect ( result . wouldExceedLimit ) . to . equal ( false )
4444 } )
4545
4646 it ( "detects when content would exceed max allowed size" , ( ) => {
4747 // Create content that would exceed max allowed size for deepseek (64k - 27k tokens)
48- const largeContent = "x" . repeat ( 148000 ) // 37k tokens > (64k - 27k) tokens
48+ const largeContent = "x" . repeat ( 148000 ) // 74k tokens > (64k - 27k) tokens
4949 const result = estimateContentSize ( largeContent , 64_000 )
5050
5151 expect ( result . wouldExceedLimit ) . to . equal ( true )
@@ -68,7 +68,7 @@ describe("content-size", () => {
6868 const result = await estimateFileSize ( tempFilePath , CONTEXT_LIMIT )
6969
7070 expect ( result . bytes ) . to . equal ( 11 )
71- expect ( result . estimatedTokens ) . to . equal ( 3 )
71+ expect ( result . estimatedTokens ) . to . equal ( 6 )
7272 expect ( result . wouldExceedLimit ) . to . equal ( false )
7373 } )
7474
0 commit comments