feat: handle super large (65535+ char) token lengths (#88)

closes #77 

very negligible impact on performance (<1%)

Author: bartveneman

src/arena.test.ts

@@ -1,6 +1,6 @@
 import { describe, test, expect } from 'vitest'
 import { readFileSync } from 'fs'
-import { CSSDataArena, STYLESHEET, STYLE_RULE, DECLARATION, FLAG_IMPORTANT, FLAG_HAS_ERROR } from './arena'
+import { CSSDataArena, STYLESHEET, STYLE_RULE, DECLARATION, FLAG_IMPORTANT, FLAG_HAS_ERROR, FLAG_LENGTH_OVERFLOW } from './arena'
 import { parse } from './parse'
 
 describe('CSSDataArena', () => {
@@ -237,6 +237,27 @@ describe('CSSDataArena', () => {
 			arena.set_flags(node, 0)
 			expect(arena.get_flags(node)).toBe(0)
 		})
+
+		test('should set FLAG_LENGTH_OVERFLOW when length > 65535', () => {
+			const arena = new CSSDataArena(10)
+			const node = arena.create_node(DECLARATION, 0, 0, 1, 1)
+
+			// Set length to exceed Uint16 max (65535)
+			arena.set_length(node, 70000)
+
+			expect(arena.get_length(node)).toBe(70000) // Should return actual length from Map
+			expect(arena.has_flag(node, FLAG_LENGTH_OVERFLOW)).toBe(true)
+		})
+
+		test('should not set FLAG_LENGTH_OVERFLOW when length <= 65535', () => {
+			const arena = new CSSDataArena(10)
+			const node = arena.create_node(DECLARATION, 0, 0, 1, 1)
+
+			arena.set_length(node, 65535)
+
+			expect(arena.get_length(node)).toBe(65535)
+			expect(arena.has_flag(node, FLAG_LENGTH_OVERFLOW)).toBe(false)
+		})
 	})
 
 	describe('growth tracking', () => {

src/arena.ts

@@ -1,6 +1,6 @@
 // CSS Data Arena - Single contiguous ArrayBuffer for all AST nodes
 //
-// Each node occupies 32 bytes with the following layout:
+// Each node occupies 36 bytes with the following layout:
 // Offset | Size | Field
 // -------|------|-------------
 //   0    |  1   | type
@@ -14,36 +14,34 @@
 //  20    |  2   | contentLength
 //  22    |  2   | valueLength
 //  24    |  4   | startLine
-//  28    |  2   | startColumn
-//  30    |  1   | attr_operator (reusing padding)
-//  31    |  1   | attr_flags (reusing padding)
+//  28    |  4   | startColumn
+//  32    |  1   | attr_operator
+//  33    |  1   | attr_flags
+//  34    |  2   | (padding)
 //
 // HOW THE ARENA WORKS:
-// 1. BYTES_PER_NODE defines the size of each node (32 bytes). The ArrayBuffer size is calculated
-//    as: capacity × BYTES_PER_NODE. For example, 1024 nodes = 32,768 bytes (32KB).
-//    Node indices map to byte offsets via: node_offset = node_index × 32.
+// 1. BYTES_PER_NODE defines the size of each node (36 bytes). The ArrayBuffer size is calculated
+//    as: capacity × BYTES_PER_NODE. For example, 1024 nodes = 36,864 bytes (36KB).
+//    Node indices map to byte offsets via: node_offset = node_index × 36.
 //
 // 2. We use a single DataView over the ArrayBuffer to read/write different types at specific offsets.
 //    - Uint8: 1-byte reads/writes for type, flags (e.g., view.getUint8(offset))
-//    - Uint16: 2-byte reads/writes for length, deltas, column (e.g., view.getUint16(offset, true))
-//    - Uint32: 4-byte reads/writes for startOffset, pointers, line (e.g., view.getUint32(offset, true))
+//    - Uint16: 2-byte reads/writes for length, deltas (e.g., view.getUint16(offset, true))
+//    - Uint32: 4-byte reads/writes for startOffset, pointers, line, column (e.g., view.getUint32(offset, true))
 //    The 'true' parameter specifies little-endian byte order (native on x86/ARM CPUs).
 //
-// 3. Padding (2 bytes total at offsets 30-31) ensures memory alignment for performance:
-//    - Uint32 fields align to 4-byte boundaries (offsets 4, 8, 12, 24)
-//    - Uint16 fields align to 2-byte boundaries (offsets 2, 16, 18, 20, 22, 28)
+// 3. Padding (2 bytes at offsets 34-35) ensures memory alignment for performance:
+//    - Uint32 fields align to 4-byte boundaries (offsets 4, 8, 12, 24, 28)
+//    - Uint16 fields align to 2-byte boundaries (offsets 2, 16, 18, 20, 22)
 //    Aligned access is faster (single CPU instruction) vs unaligned (multiple memory accesses).
 //    Modern CPUs penalize unaligned reads/writes, making padding essential for performance.
 //
-// 4. The padding at offset 30-31 is reused for attribute selector data (attr_operator, attr_flags),
-//    making efficient use of otherwise wasted bytes. This is a space optimization trick.
-//
-// 5. Delta offsets (contentStartDelta, valueStartDelta) save memory: instead of storing absolute
+// 4. Delta offsets (contentStartDelta, valueStartDelta) save memory: instead of storing absolute
 //    positions as uint32 (4 bytes), we store relative offsets as uint16 (2 bytes). Removing unused
-//    lastChild field saved another 4 bytes. This reduced node size from 44→40→36→32 bytes (27%
-//    smaller than original), saving memory while maintaining performance.
+//    lastChild field saved another 4 bytes. startColumn was changed from Uint16 to Uint32 to avoid
+//    overflow on long lines (common in minified CSS). Node size: 44→40→36 bytes.
 
-let BYTES_PER_NODE = 32
+let BYTES_PER_NODE = 36
 
 // Node type constants
 export const STYLESHEET = 1
@@ -119,6 +117,7 @@ export class CSSDataArena {
 	private capacity: number // Number of nodes that can fit
 	private count: number // Number of nodes currently allocated
 	private growth_count: number // Number of times the arena has grown
+	private overflow_lengths: Map<number, number> // Stores actual lengths for nodes > 65535 chars
 
 	// Growth multiplier when capacity is exceeded
 	private static readonly GROWTH_FACTOR = 1.3
@@ -136,6 +135,7 @@ export class CSSDataArena {
 		this.growth_count = 0
 		this.buffer = new ArrayBuffer(initial_capacity * BYTES_PER_NODE)
 		this.view = new DataView(this.buffer)
+		this.overflow_lengths = new Map()
 	}
 
 	// Calculate recommended initial capacity based on CSS source size
@@ -185,6 +185,13 @@ export class CSSDataArena {
 
 	// Read length in source
 	get_length(node_index: number): number {
+		// Check if this node has overflow length stored
+		if (this.has_flag(node_index, FLAG_LENGTH_OVERFLOW)) {
+			const overflow_length = this.overflow_lengths.get(node_index)
+			if (overflow_length !== undefined) {
+				return overflow_length
+			}
+		}
 		return this.view.getUint16(this.node_offset(node_index) + 2, true)
 	}
 
@@ -202,12 +209,12 @@ export class CSSDataArena {
 
 	// Read attribute operator (for NODE_SELECTOR_ATTRIBUTE)
 	get_attr_operator(node_index: number): number {
-		return this.view.getUint8(this.node_offset(node_index) + 30)
+		return this.view.getUint8(this.node_offset(node_index) + 32)
 	}
 
 	// Read attribute flags (for NODE_SELECTOR_ATTRIBUTE)
 	get_attr_flags(node_index: number): number {
-		return this.view.getUint8(this.node_offset(node_index) + 31)
+		return this.view.getUint8(this.node_offset(node_index) + 33)
 	}
 
 	// Read first child index (0 = no children)
@@ -227,7 +234,7 @@ export class CSSDataArena {
 
 	// Read start column
 	get_start_column(node_index: number): number {
-		return this.view.getUint16(this.node_offset(node_index) + 28, true)
+		return this.view.getUint32(this.node_offset(node_index) + 28, true)
 	}
 
 	// Read value start offset (stored as delta from startOffset, declaration value / at-rule prelude)
@@ -254,14 +261,17 @@ export class CSSDataArena {
 		this.view.setUint8(this.node_offset(node_index) + 1, flags)
 	}
 
-	// Write start offset in source
-	set_start_offset(node_index: number, offset: number): void {
-		this.view.setUint32(this.node_offset(node_index) + 12, offset, true)
-	}
-
 	// Write length in source
 	set_length(node_index: number, length: number): void {
-		this.view.setUint16(this.node_offset(node_index) + 2, length, true)
+		// Uint16 max value is 65535
+		if (length > 65535) {
+			this.view.setUint16(this.node_offset(node_index) + 2, 65535, true)
+			this.set_flag(node_index, FLAG_LENGTH_OVERFLOW)
+			// Store the actual length in the overflow map
+			this.overflow_lengths.set(node_index, length)
+		} else {
+			this.view.setUint16(this.node_offset(node_index) + 2, length, true)
+		}
 	}
 
 	// Write content start delta (offset from startOffset)
@@ -276,12 +286,12 @@ export class CSSDataArena {
 
 	// Write attribute operator (for NODE_SELECTOR_ATTRIBUTE)
 	set_attr_operator(node_index: number, operator: number): void {
-		this.view.setUint8(this.node_offset(node_index) + 30, operator)
+		this.view.setUint8(this.node_offset(node_index) + 32, operator)
 	}
 
 	// Write attribute flags (for NODE_SELECTOR_ATTRIBUTE)
 	set_attr_flags(node_index: number, flags: number): void {
-		this.view.setUint8(this.node_offset(node_index) + 31, flags)
+		this.view.setUint8(this.node_offset(node_index) + 33, flags)
 	}
 
 	// Write first child index
@@ -294,16 +304,6 @@ export class CSSDataArena {
 		this.view.setUint32(this.node_offset(node_index) + 8, siblingIndex, true)
 	}
 
-	// Write start line
-	set_start_line(node_index: number, line: number): void {
-		this.view.setUint32(this.node_offset(node_index) + 24, line, true)
-	}
-
-	// Write start column
-	set_start_column(node_index: number, column: number): void {
-		this.view.setUint16(this.node_offset(node_index) + 28, column, true)
-	}
-
 	// Write value start delta (offset from startOffset, declaration value / at-rule prelude)
 	set_value_start_delta(node_index: number, delta: number): void {
 		this.view.setUint16(this.node_offset(node_index) + 18, delta, true)
@@ -342,10 +342,12 @@ export class CSSDataArena {
 
 		const offset = node_index * BYTES_PER_NODE
 		this.view.setUint8(offset, type) // +0: type
-		this.view.setUint16(offset + 2, length, true) // +2: length
 		this.view.setUint32(offset + 12, start_offset, true) // +12: startOffset
 		this.view.setUint32(offset + 24, start_line, true) // +24: startLine
-		this.view.setUint16(offset + 28, start_column, true) // +28: startColumn
+		this.view.setUint32(offset + 28, start_column, true) // +28: startColumn
+
+		// Use setter method to handle overflow
+		this.set_length(node_index, length)
 
 		return node_index
 	}

src/parse.test.ts

@@ -12,6 +12,7 @@ import {
 	TYPE_SELECTOR,
 	ATTRIBUTE_SELECTOR,
 	NESTING_SELECTOR,
+	URL,
 } from './constants'
 import { ATTR_OPERATOR_PIPE_EQUAL } from './arena'
 
@@ -2508,4 +2509,65 @@ describe('Core Nodes', () => {
 			})
 		})
 	})
+
+	describe('Large inline SVG', () => {
+		test('should correctly parse declaration with huge inline SVG background-image', () => {
+			// Generate a very long SVG string (> 65535 chars)
+			const svgPart = '<svg xmlns="http://www.w3.org/2000/svg"><rect width="100" height="100" fill="red"/></svg>'
+			const longSvg = svgPart.repeat(1000) // 89,000 chars
+			// Add a second declaration after the huge SVG to test startColumn overflow
+			const css = `.test { background-image: url("data:image/svg+xml,${longSvg}"); color: red; }`
+
+			expect(longSvg.length).toBeGreaterThan(65535) // Verify SVG is long enough
+
+			const ast = parse(css)
+			const rule = ast.first_child!
+			const block = rule.block!
+			const declaration = block.first_child!
+
+			// Verify declaration is parsed correctly
+			expect(declaration.type).toBe(DECLARATION)
+			expect(declaration.property).toBe('background-image')
+
+			// Verify the full length is accessible (not truncated)
+			const declText = `background-image: url("data:image/svg+xml,${longSvg}");`
+			expect(declaration.length).toBe(declText.length)
+			expect(declaration.length).toBeGreaterThan(65535)
+
+			// Verify we can access the full declaration text
+			expect(declaration.text).toBe(declText)
+			expect(declaration.text).toContain('background-image:')
+			expect(declaration.text).toContain(longSvg.substring(0, 100))
+			expect(declaration.text).toContain(longSvg.substring(longSvg.length - 100))
+
+			// Verify the value is parsed into nodes
+			const urlNode = declaration.first_child!
+			expect(urlNode.type).toBe(URL)
+			expect(urlNode.name).toBe('url')
+
+			// Verify the URL node text (full url(...) including function name and parens)
+			const expectedUrlText = `url("data:image/svg+xml,${longSvg}")`
+			expect(urlNode.text).toBe(expectedUrlText)
+			expect(urlNode.text.length).toBe(expectedUrlText.length)
+
+			// Verify the URL node length matches its text length
+			expect(urlNode.length).toBe(urlNode.text.length)
+			expect(urlNode.length).toBe(expectedUrlText.length)
+			expect(urlNode.length).toBeGreaterThan(65535)
+
+			// Test startColumn overflow: second declaration starts at column > 65535
+			const secondDecl = declaration.next_sibling!
+			expect(secondDecl).toBeTruthy()
+			expect(secondDecl.type).toBe(DECLARATION)
+			expect(secondDecl.property).toBe('color')
+			expect(secondDecl.value).toBe('red')
+
+			// Calculate expected column: '.test { ' + declaration.text + ' ' + 1 (columns are 1-indexed)
+			const expectedColumn = '.test { '.length + declText.length + ' '.length + 1
+			expect(expectedColumn).toBeGreaterThan(65535)
+
+			// Verify column is correctly stored (Uint32, no overflow needed)
+			expect(secondDecl.column).toBe(expectedColumn)
+		})
+	})
 })