Tighten free-string budget so we don’t exhaust the token budget on multi-string structures

mattt · mattt · commit c0cc2a3e05fd · 2026-02-03T06:20:43.000-08:00
diff --git a/Sources/AnyLanguageModel/StructuredGeneration.swift b/Sources/AnyLanguageModel/StructuredGeneration.swift
@@ -150,8 +150,9 @@ struct ConstrainedJSONGenerator<Backend: TokenBackend> {
     }
 
     private func maxFreeStringTokens() -> Int {
-        let perStringLimit = max(32, backend.totalTokenBudget / 4)
-        return min(backend.remainingTokens, perStringLimit)
+        let perStringLimit = max(16, backend.totalTokenBudget / 16)
+        let remainingAfterClosingQuote = max(0, backend.remainingTokens - 1)
+        return min(remainingAfterClosingQuote, perStringLimit)
     }
 
     private mutating func generateFreeString(maxTokens: Int) throws -> String {

Original file line number	Diff line number	Diff line change
`@@ -150,8 +150,9 @@ struct ConstrainedJSONGenerator<Backend: TokenBackend> {`
`150`	`150`	`}`
`151`	`151`
`152`	`152`	`private func maxFreeStringTokens() -> Int {`
`153`		`- let perStringLimit = max(32, backend.totalTokenBudget / 4)`
`154`		`- return min(backend.remainingTokens, perStringLimit)`
	`153`	`+ let perStringLimit = max(16, backend.totalTokenBudget / 16)`
	`154`	`+ let remainingAfterClosingQuote = max(0, backend.remainingTokens - 1)`
	`155`	`+ return min(remainingAfterClosingQuote, perStringLimit)`
`155`	`156`	`}`
`156`	`157`
`157`	`158`	`private mutating func generateFreeString(maxTokens: Int) throws -> String {`