feat: add tokenization infrastructure for improved line breaking

MISSING_FEATURES.md

@@ -252,4 +252,4 @@ All tests use the `MTMathListBuilder.build(fromString:error:)` API and automatic
 
 *Generated: 2025-10-01*
 *SwiftMath Version: Based on iosMath v0.9.5*
-*Last Updated: 2025-10-01 - Implemented 4 major features: \substack, \smallmatrix, starred matrices, \iiiint*
+*Last Updated: 2026-01-07 - All features from October 2025 remain current. Recent work has focused on line breaking improvements and tokenization infrastructure.*

README.md

@@ -315,65 +315,68 @@ label.preferredMaxLayoutWidth = 150
 ```swift
 label.latex = "a+\\frac{1}{2}+\\sqrt{3}+b"
 label.preferredMaxLayoutWidth = 200
-// Intelligently breaks between complex mathematical elements
+// Breaks between complex mathematical elements
 ```
 
 #### Limited Support Cases
 
 These cases work but with some constraints:
 
-**⚠️ Atoms with superscripts/subscripts:**
+**✅ Large operators (NEW!):**
 ```swift
-label.latex = "a^{2}+b^{2}+c^{2}+d^{2}+e^{2}"
-label.preferredMaxLayoutWidth = 150
-// Works, but uses fallback breaking mechanism
-// May not break at the most optimal positions
+label.latex = "a + \\sum_{i=1}^{n} x_i + \\int_{0}^{1} f(x)dx + b"
+label.preferredMaxLayoutWidth = 200
+// Large operators stay inline when they fit!
+// Includes height checking for operators with limits
 ```
-**Note**: Scripted atoms (with superscripts/subscripts) trigger the universal breaking mechanism which breaks within accumulated text rather than at atom boundaries. This still works but may not be as clean as pure interatom breaking.
 
-**⚠️ Very long single text atoms:**
+**✅ Delimited expressions (NEW!):**
 ```swift
-label.latex = "\\text{This is an extremely long piece of text within a single text command}"
+label.latex = "(a+b) + \\left(\\frac{c}{d}\\right) + e"
 label.preferredMaxLayoutWidth = 200
-// Uses Unicode word boundary breaking with Core Text
-// Protects numbers from being split (e.g., "3.14" stays together)
+// Delimiters stay inline when they fit!
+// Inner content respects width constraints and wraps naturally
 ```
 
-#### Remaining Unsupported Cases
-
-These atom types still force line breaks (not yet optimized):
-
-**⚠️ Large operators (∑, ∫, ∏, lim):**
+**✅ Colored expressions (NEW!):**
 ```swift
-label.latex = "\\sum_{i=1}^{n} x_i + \\int_{0}^{1} f(x)dx"
-// Each operator forces a new line
+label.latex = "a + \\color{red}{b + c + d} + e"
+label.preferredMaxLayoutWidth = 200
+// Colored sections stay inline when they fit!
+// Inner content respects width constraints and wraps properly
 ```
 
-**⚠️ Matrices and tables:**
+**✅ Matrices/tables (NEW!):**
 ```swift
-label.latex = "A = \\begin{pmatrix} 1 & 2 \\\\ 3 & 4 \\end{pmatrix}"
-// Matrix always on own line
+label.latex = "A = \\begin{pmatrix} 1 & 2 \\end{pmatrix} + B"
+label.preferredMaxLayoutWidth = 200
+// Small matrices stay inline when they fit!
 ```
 
-**⚠️ Delimited expressions (\left...\right):**
+**✅ Atoms with superscripts/subscripts (IMPROVED!):**
 ```swift
-label.latex = "\\left(\\frac{a}{b}\\right) + c"
-// The parenthesized group forces line breaks
+label.latex = "a^{2}+b^{2}+c^{2}+d^{2}+e^{2}"
+label.preferredMaxLayoutWidth = 150
+// Now works with width-based breaking!
+// Scripted atoms participate in smart line breaking decisions
 ```
 
-**⚠️ Colored expressions:**
+**✅ Math accents:**
 ```swift
-label.latex = "a + \\color{red}{b} + c"
-// Colored portion causes line break
+label.latex = "\\hat{x} + \\tilde{y} + \\bar{z} + \\vec{w}"
+label.preferredMaxLayoutWidth = 150
+// Common accents (\hat, \tilde, \bar, \vec) work well
+// Vector arrows (\overrightarrow, etc.) supported with stretching
 ```
 
-**⚠️ Math accents (partial support):**
+**⚠️ Very long single text atoms:**
 ```swift
-label.latex = "\\hat{x} + \\tilde{y} + \\bar{z}"
-// Common accents (\hat, \tilde, \bar) are positioned correctly in most cases.
-// Some complex grapheme clusters or font-specific metrics may still need additional polishing.
-// See MULTILINE_IMPLEMENTATION_NOTES.md for details and known edge cases.
+label.latex = "\\text{This is an extremely long piece of text within a single text command}"
+label.preferredMaxLayoutWidth = 200
+// Uses Unicode word boundary breaking with Core Text
+// Protects numbers from being split (e.g., "3.14" stays together)
 ```
+**Note**: Breaks within the text atom rather than between atoms, which is expected behavior for very long continuous text.
 
 #### Best Practices
 
@@ -409,7 +412,7 @@ label.preferredMaxLayoutWidth = 150
 label.latex = "a+\\frac{1}{2}+b+\\frac{3}{4}+c"
 label.preferredMaxLayoutWidth = 200
 // ✅ Fractions stay inline when they fit!
-// Breaks intelligently: "a + ½ + b" on line 1, "+ ¾ + c" on line 2
+// Breaks: "a + ½ + b" on line 1, "+ ¾ + c" on line 2
 ```
 
 **Excellent use case (radicals inline - NEW!):**
@@ -430,11 +433,14 @@ label.preferredMaxLayoutWidth = 0  // No breaking
 
 #### Technical Details
 
-- **Line spacing**: New lines are positioned at `fontSize × 1.5` below the previous line
-- **Breaking algorithm**: Greedy - breaks immediately when projected width exceeds constraint
+- **Line spacing**: Dynamic line height based on actual content (tall fractions get more space, regular content stays compact)
+- **Breaking algorithm**: Greedy with look-ahead and break quality scoring - prefers breaking after operators over other positions
 - **Width calculation**: Includes inter-element spacing according to TeX spacing rules
 - **Number protection**: Numbers in patterns like "3.14", "1,000", etc. are kept intact
 - **Supports locales**: English, French, Swiss number formats
+- **Advanced features**: Tokenization infrastructure with phase-based processing (preprocessing → tokenization → line fitting → display generation)
+
+For complete implementation details including recent improvements (dynamic line height, break quality scoring, early exit optimization), see [MULTILINE_IMPLEMENTATION_NOTES.md](MULTILINE_IMPLEMENTATION_NOTES.md).
 
 ### Included Features
 This is a list of formula types that the library currently supports:
@@ -453,7 +459,10 @@ This is a list of formula types that the library currently supports:
 * Ratios, proportions, percentages
 * Math spacing
 * Overline and underline
-* Math accents
+* Math accents (including `\hat`, `\tilde`, `\bar`, `\vec`, `\dot`, `\ddot`, etc.)
+* Vector arrows (`\vec`, `\overrightarrow`, `\overleftarrow`, `\overleftrightarrow` with automatic stretching)
+* Wide accents (`\widehat`, `\widetilde`)
+* Operator names (`\operatorname{name}`)
 * Matrices (including `\smallmatrix` and starred variants like `pmatrix*` with alignment)
 * Multi-line subscripts and limits (`\substack`)
 * Equation alignment
@@ -462,6 +471,7 @@ This is a list of formula types that the library currently supports:
 * Most commonly used math symbols
 * Colors for both text and background
 * **Inline and display math mode delimiters** (see below)
+* **Automatic line wrapping** (see Automatic Line Wrapping section)
 
 ### LaTeX Math Delimiters
 

Sources/SwiftMath/MathBundle/MTFontMathTableV2.swift

@@ -79,8 +79,13 @@ internal class MTFontMathTableV2: MTFontMathTable {
     }
     /** Returns a larger vertical variant of the given glyph if any.
      If there is no larger version, this returns the current glyph.
+
+     - Parameter glyph: The glyph to find a larger variant for
+     - Parameter forDisplayStyle: If true, selects the largest appropriate variant for display style.
+                                  If false, selects the next larger variant (incremental sizing).
+     - Returns: A larger glyph variant, or the original glyph if no variants exist
      */
-    override func getLargerGlyph(_ glyph: CGGlyph) -> CGGlyph {
+    override func getLargerGlyph(_ glyph: CGGlyph, forDisplayStyle: Bool = false) -> CGGlyph {
         let font = mathFont.mtfont(size: fontSize)
         let glyphName = font.get(nameForGlyph: glyph)
 
@@ -89,13 +94,42 @@ internal class MTFontMathTableV2: MTFontMathTable {
             // There are no extra variants, so just returnt the current glyph.
             return glyph
         }
-        // Find the first variant with a different name.
-        for gvn in variantGlyphs {
-            if let glyphVariantName = gvn as? String, glyphVariantName != glyphName {
+
+        if forDisplayStyle {
+            // For display style, select a large variant suitable for mathematical display mode
+            // Display integrals should be significantly larger (~2.2em) for visual prominence
+            let count = variantGlyphs.count
+
+            // Strategy: Use the largest variant, but avoid extreme sizes for fonts with many variants
+            let targetIndex: Int
+            if count <= 2 {
+                // Small variant list: use the last one (e.g., integral.size1 at ~2.2em)
+                targetIndex = count - 1
+            } else if count <= 4 {
+                // Medium variant list: use second-to-last to avoid extremes
+                targetIndex = count - 2
+            } else {
+                // Large variant list (like texgyretermes with 6 variants):
+                // Use variant at ~60% position to get appropriate display size (~2.0em)
+                // For 7 variants (0-6), this gives index 4
+                targetIndex = min(count - 2, Int(Double(count) * 0.6))
+            }
+
+            if let glyphVariantName = variantGlyphs[targetIndex] as? String {
                 let variantGlyph = font.get(glyphWithName: glyphVariantName)
                 return variantGlyph
             }
+        } else {
+            // Text/inline style: use incremental sizing for moderate enlargement
+            // Find the first variant with a different name
+            for gvn in variantGlyphs {
+                if let glyphVariantName = gvn as? String, glyphVariantName != glyphName {
+                    let variantGlyph = font.get(glyphWithName: glyphVariantName)
+                    return variantGlyph
+                }
+            }
         }
+
         // We did not find any variants of this glyph so return it.
         return glyph
     }

Sources/SwiftMath/MathRender/MTFontMathTable.swift

@@ -103,16 +103,15 @@ class MTFontMathTable {
     /// specified in the OpenType Math specification. Rather these are proposed LuaTeX extensions
     /// for the TeX parameters \sigma_20 (delim1) and \sigma_21 (delim2). Since these do not
     /// exist in the fonts that we have, we use the same approach as LuaTeX and use the fontSize
-    /// to determine these values. The constants used are the same as LuaTeX and KaTeX and match the
-    /// metrics values of the original TeX fonts.
+    /// to determine these values. The constants used match the metrics values of the original TeX fonts.
     /// Note: An alternative approach is to use DelimitedSubFormulaMinHeight for \sigma21 and use a factor
     /// of 2 to get \sigma 20 as proposed in Vieth paper.
     /// The XeTeX implementation sets \sigma21 = fontSize and \sigma20 = DelimitedSubFormulaMinHeight which
     /// will produce smaller delimiters.
     /// Of all the approaches we've implemented LuaTeX's approach since it mimics LaTeX most accurately.
     var fractionDelimiterSize: CGFloat { 1.01 * _fontSize }
-    
-    /// Modified constant from 2.4 to 2.39, it matches KaTeX and looks better.
+
+    /// Modified constant from 2.4 to 2.39 for better visual appearance.
     var fractionDelimiterDisplayStyleSize: CGFloat { 2.39 * _fontSize }
 
     // MARK: - Stacks
@@ -220,23 +219,57 @@ class MTFontMathTable {
 
     /** Returns a larger vertical variant of the given glyph if any.
      If there is no larger version, this returns the current glyph.
+
+     - Parameter glyph: The glyph to find a larger variant for
+     - Parameter forDisplayStyle: If true, selects the largest appropriate variant for display style.
+                                  If false, selects the next larger variant (incremental sizing).
+     - Returns: A larger glyph variant, or the original glyph if no variants exist
      */
-    func getLargerGlyph(_ glyph:CGGlyph) -> CGGlyph {
+    func getLargerGlyph(_ glyph:CGGlyph, forDisplayStyle: Bool = false) -> CGGlyph {
         let variants = _mathTable[kVertVariants] as! NSDictionary?
         let glyphName = self.font?.get(nameForGlyph: glyph)
         let variantGlyphs = variants![glyphName!] as! NSArray?
         if variantGlyphs == nil || variantGlyphs?.count == 0 {
             // There are no extra variants, so just returnt the current glyph.
             return glyph
         }
-        // Find the first variant with a different name.
-        for gvn in variantGlyphs! {
-            let glyphVariantName = gvn as! String?
-            if glyphVariantName != glyphName {
-                let variantGlyph = self.font?.get(glyphWithName: glyphVariantName!)
+
+        if forDisplayStyle {
+            // For display style, select a large variant suitable for mathematical display mode
+            // Display integrals should be significantly larger (~2.2em) for visual prominence
+            let count = variantGlyphs!.count
+
+            // Strategy: Use the largest variant, but avoid extreme sizes for fonts with many variants
+            let targetIndex: Int
+            if count <= 2 {
+                // Small variant list: use the last one (e.g., integral.size1 at ~2.2em)
+                targetIndex = count - 1
+            } else if count <= 4 {
+                // Medium variant list: use second-to-last to avoid extremes
+                targetIndex = count - 2
+            } else {
+                // Large variant list (like texgyretermes with 6 variants):
+                // Use variant at ~60% position to get appropriate display size (~2.0em)
+                // For 7 variants (0-6), this gives index 4
+                targetIndex = min(count - 2, Int(Double(count) * 0.6))
+            }
+
+            if let glyphVariantName = variantGlyphs![targetIndex] as? String {
+                let variantGlyph = self.font?.get(glyphWithName: glyphVariantName)
                 return variantGlyph!
             }
+        } else {
+            // Text/inline style: use incremental sizing for moderate enlargement
+            // Find the first variant with a different name
+            for gvn in variantGlyphs! {
+                let glyphVariantName = gvn as! String?
+                if glyphVariantName != glyphName {
+                    let variantGlyph = self.font?.get(glyphWithName: glyphVariantName!)
+                    return variantGlyph!
+                }
+            }
         }
+
         // We did not find any variants of this glyph so return it.
         return glyph;
     }

Sources/SwiftMath/MathRender/MTMathList.swift

@@ -922,7 +922,23 @@ public class MTMathList : NSObject {
             let newNode = atom.finalized
 
             if NSEqualRanges(zeroRange, atom.indexRange) {
-                let index = prevNode == nil ? 0 : prevNode!.indexRange.location + prevNode!.indexRange.length
+                // CRITICAL FIX: Check if prevNode has a valid range location before using it
+                // If location is NSNotFound, treat as if there's no prevNode
+                // This prevents negative overflow when creating NSMakeRange
+                let index: Int
+                if prevNode == nil || prevNode!.indexRange.location == NSNotFound {
+                    index = 0
+                } else {
+                    // Additional safety: check for potential overflow
+                    let location = prevNode!.indexRange.location
+                    let length = prevNode!.indexRange.length
+                    // If either value is suspicious (negative or too large), reset to 0
+                    if location < 0 || length < 0 || location > Int.max - length {
+                        index = 0
+                    } else {
+                        index = location + length
+                    }
+                }
                 newNode.indexRange = NSMakeRange(index, 1)
             }
 

Sources/SwiftMath/MathRender/MTMathListBuilder.swift

@@ -742,9 +742,11 @@ public struct MTMathListBuilder {
             skipSpaces()
             if hasCharacters && string[currentCharIndex] == "[" {
                 _ = getNextCharacter() // consume '['
-                let alignmentChar = getNextCharacter()
-                if alignmentChar == "l" || alignmentChar == "r" || alignmentChar == "c" {
-                    frac.alignment = String(alignmentChar)
+                if hasCharacters {
+                    let alignmentChar = getNextCharacter()
+                    if alignmentChar == "l" || alignmentChar == "r" || alignmentChar == "c" {
+                        frac.alignment = String(alignmentChar)
+                    }
                 }
                 // Consume closing ']'
                 if hasCharacters && string[currentCharIndex] == "]" {
@@ -1116,9 +1118,11 @@ public struct MTMathListBuilder {
             skipSpaces()
             if hasCharacters && string[currentCharIndex] == "[" {
                 _ = getNextCharacter() // consume '['
-                let alignmentChar = getNextCharacter()
-                if alignmentChar == "l" || alignmentChar == "r" || alignmentChar == "c" {
-                    frac.alignment = String(alignmentChar)
+                if hasCharacters {
+                    let alignmentChar = getNextCharacter()
+                    if alignmentChar == "l" || alignmentChar == "r" || alignmentChar == "c" {
+                        frac.alignment = String(alignmentChar)
+                    }
                 }
                 // Consume closing ']'
                 if hasCharacters && string[currentCharIndex] == "]" {
@@ -1166,6 +1170,10 @@ public struct MTMathListBuilder {
             return frac
         } else if command == "sqrt" {
             let rad = MTRadical()
+            guard self.hasCharacters else {
+                rad.radicand = self.buildInternal(true)
+                return rad
+            }
             let char = self.getNextCharacter()
             if char == "[" {
                 rad.degree = self.buildInternal(false, stopChar: "]")