Tokenization/chars+escapes

examples/AdventOfCode2025/d3_batteries.mx

@@ -119,11 +119,11 @@
 		%banksSum(strar)
 	}
 	%print_endl(
-		outc 10
-		outc 83
-		outc 117
-		outc 109
-		outc 32
+		outc '\n'
+		outc 'S'
+		outc 'u'
+		outc 'm'
+		outc ' '
 		%banksSum(outum)
 	)
 }

examples/fibonacci_asm_only.mx

@@ -1,14 +1,14 @@
 ; input prompt
-outc 72
-outc 111
-outc 119
-outc 32
-outc 109
-outc 97
-outc 110
-outc 121
-outc 63
-outc 32
+outc 'H'
+outc 'o'
+outc 'w'
+outc ' '
+outc 'm'
+outc 'a'
+outc 'n'
+outc 'y'
+outc '?'
+outc ' '
 
 inu ; input
 inl
@@ -30,7 +30,7 @@ strr
 ; print a
 mov 0
 outum
-outc 10
+outc '\n'
 
 ; (a, b) = (b, a + b)
 ldm

std/debug.mx

@@ -31,7 +31,7 @@
 %macro emitMemdumpAddr() {
 	outc 10
 	outuh
-	outc 58
+	outc ':'
 }
 
 ; from, count -> void (stdout)
@@ -88,7 +88,7 @@
 %macro dumpStackOverrun(overrun) {
 	%_dumpStack2Top()
 	; show overrun
-	outc 124 ; separator
+	outc '|' ; separator
 	%load(%SEG_DEBUG_GLOBALS)
 	lda 1
 	%stack:pushr() ; from original SP
@@ -101,9 +101,9 @@
 %macro seeFuncFrame() {
 	; TODO factor to function
 	%toggleDebugMode()
-	outc 10
-	outc 70
-	outc 58
+	outc '\n'
+	outc 'F'
+	outc ':'
 
 	%load(!op(a, %SEG_DEBUG_GLOBALS, %G_ARGS_PTR)) ; memdump normal args
 	%stack:pushr()
@@ -114,7 +114,7 @@
 	%stack:pushr()
 	%if_else { ldm ,
 		%call(memdump)
-		outc 124
+		outc '|'
 	,
 		%stack:dropN(2)
 	}

std/ds/array.mx

@@ -244,11 +244,11 @@
 	%void_method{ array_print, 1, 1,
 		; locals: first | for_stop, for_idx
 		%seg:local(0, str 1)
-		outc 91
+		outc '['
 		%foreach {
 			%if_else { %seg:local(0, lmeq 0),
-				outc 44
-				outc 32
+				outc ','
+				outc ' '
 			,
 				str 0
 			}
@@ -258,8 +258,8 @@
 				outum
 			)
 		}
-		outc 93
-		outc 10
+		outc ']'
+		outc '\n'
 	}
 
 

std/exceptions.mx

@@ -3,16 +3,16 @@
 %define ERR_ASSERT 0
 
 %macro error(code) {
-	outc 10
-	outc 69
-	outc 82
-	outc 82
-	outc 79
-	outc 82
-	outc 58
-	outc 32
+	outc '\n'
+	outc 'E'
+	outc 'R'
+	outc 'R'
+	outc 'O'
+	outc 'R'
+	outc ':'
+	outc ' '
 	outu %code
-	outc 10
+	outc '\n'
 	jmp end
 }
 

std/io.mx

@@ -3,19 +3,19 @@
 
 %macro print_spaced(instr) {
 	%instr
-	outc 32
+	outc ' '
 }
 %macro print_endl(instr) {
 	%instr
-	outc 10
+	outc '\n'
 }
 %macro print_separator() {
-	outc 44
-	outc 32
+	outc ','
+	outc ' '
 }
 %macro print_key_val(keyChar, val) {
 	outc %keyChar
-	outc 61
+	outc '='
 	outu(%val)
 	%print_separator()
 }
@@ -26,7 +26,7 @@
 	%if {
 		ld> 15 ; negative?
 	,
-		outc 45
+		outc '-'
 		ld 0
 		ldsm
 		strr
@@ -56,18 +56,19 @@
 
 ; VM / runtime delim
 %macro print_delim(vm) {
-	outc 45
-	outc 32
+	outc '-'
+	outc ' '
 	%if_else(ld %vm,
-		outc 86
-		outc 77,
-		outc 82
-		outc 85
-		outc 78
+		outc 'V'
+		outc 'M'
+	,
+		outc 'R'
+		outc 'U'
+		outc 'N'
 	)
-	outc 32
-	outc 45
-	outc 10
+	outc ' '
+	outc '-'
+	outc '\n'
 }
 %macro vm_runtime_delim() {
 	ld !print_delim(1)

tests/chars.mx

@@ -0,0 +1,65 @@
+
+';'
+'%'
+"%"
+'!'
+"!"
+'$'
+"$"
+','
+","
+':'
+":"
+'('
+"("
+
+"
+'
+''
+""
+' '
+'"'
+'\
+'\'
+'\\
+'\\'
+'\0'
+"\r\n\0"
+
+'\''
+'\"'
+'"'
+"\""
+"'"
+"\'"
+"""
+'''
+lll ' ' ' '
+
+';';comment
+;';';comment
+';;comment
+\;
+"\;
+
+; complex fields
+%(def('i')ne) ["LMA\0"]
+%(def('i')ne) [("LMAO")] ('^')
+%define escaped '\r'
+out('u') '\n'
+
+%macro testValues(defineName) {
+	; OMG I am so HIGH ~~
+	outu '8'
+	outc '\''
+	outu  8
+	outc '"'
+	outc '\t'
+	outc  8 ; probably some bullshit
+	outc '\n'
+	outc '8'
+	outc %(%defineName)
+	outu %("escaped")
+	out('c') '\n'
+}
+ld !testValues("LMAO")

tests/chars.txt

@@ -0,0 +1,51 @@
+:returncode 1
+
+:stdout 13
+56'8"	
+8^13
+
+
+:stderr 2149
+tests\chars.mx:16:1 ERROR: Expected string or character termination
+tests\chars.mx:17:1 ERROR: Expected string or character termination
+tests\chars.mx:18:1 ERROR: Invalid character value
+tests\chars.mx:22:1 ERROR: Invalid escape sequence '\'
+tests\chars.mx:23:1 ERROR: Expected string or character termination
+tests\chars.mx:24:1 ERROR: Expected string or character termination
+tests\chars.mx:35:3 ERROR: Expected string or character termination
+tests\chars.mx:36:1 ERROR: Invalid character value
+tests\chars.mx:36:3 ERROR: Expected string or character termination
+tests\chars.mx:41:1 ERROR: Expected string or character termination
+tests\chars.mx:43:1 ERROR: Invalid escape sequence '\;'
+tests\chars.mx:46:15 ERROR: Invalid define name 'LMA\0'
+tests\chars.mx:2:1 ERROR: Unexpected token ';'
+tests\chars.mx:3:1 ERROR: Unexpected token '%'
+tests\chars.mx:4:1 ERROR: Unexpected token "%"
+tests\chars.mx:5:1 ERROR: Unexpected token '!'
+tests\chars.mx:6:1 ERROR: Unexpected token "!"
+tests\chars.mx:7:1 ERROR: Unexpected token '$'
+tests\chars.mx:8:1 ERROR: Unexpected token "$"
+tests\chars.mx:9:1 ERROR: Unexpected token ','
+tests\chars.mx:10:1 ERROR: Unexpected token ","
+tests\chars.mx:11:1 ERROR: Unexpected token ':'
+tests\chars.mx:12:1 ERROR: Unexpected token ":"
+tests\chars.mx:13:1 ERROR: Unexpected token '('
+tests\chars.mx:14:1 ERROR: Unexpected token "("
+tests\chars.mx:19:1 ERROR: Unexpected token ""
+tests\chars.mx:20:1 ERROR: Unexpected token ' '
+tests\chars.mx:21:1 ERROR: Unexpected token '"'
+tests\chars.mx:25:1 ERROR: Unexpected token '\\'
+tests\chars.mx:26:1 ERROR: Unexpected token '\0'
+tests\chars.mx:27:1 ERROR: Unexpected token "\r\n\0"
+tests\chars.mx:29:1 ERROR: Unexpected token '\''
+tests\chars.mx:30:1 ERROR: Unexpected token '\"'
+tests\chars.mx:31:1 ERROR: Unexpected token '"'
+tests\chars.mx:32:1 ERROR: Unexpected token "\""
+tests\chars.mx:33:1 ERROR: Unexpected token "'"
+tests\chars.mx:34:1 ERROR: Unexpected token "\'"
+tests\chars.mx:35:1 ERROR: Unexpected token ""
+tests\chars.mx:39:1 ERROR: Unexpected token ';'
+tests\chars.mx:42:1 ERROR: Unexpected token '\'
+tests\chars.mx:37:1 ERROR: Unknown condition 'lll ' ' ' ''
+
+

tests/complex-names.txt

@@ -1,6 +1,6 @@
 :returncode 1
 
-:stderr 2723
+:stderr 2728
 tests\complex-names.mx:41:4 ERROR: Directive must not continue
 tests\complex-names.mx:42:5 ERROR: Directive must not continue
 tests\complex-names.mx:5:2 ERROR: Expected label field
@@ -17,7 +17,7 @@ tests\complex-names.mx:22:5 ERROR: Expected label field
 tests\complex-names.mx:25:1 ERROR: Invalid label name 'n8$5'
 tests\complex-names.mx:29:1 ERROR: Invalid label name '7zaza'
 tests\complex-names.mx:30:2 ERROR: Simple label field expected
-tests\complex-names.mx:31:1 ERROR: Invalid label name 'a:b$(,)$'
+tests\complex-names.mx:31:1 ERROR: Invalid label name 'a:b$()$'
 tests\complex-names.mx:34:5 ERROR: Expected immediate field
 tests\complex-names.mx:35:4 ERROR: Expected instr field
 tests\complex-names.mx:36:1 ERROR: Missing immediate field name
@@ -28,7 +28,7 @@ tests\complex-names.mx:43:3 ERROR: Simple instr field expected
 tests\complex-names.mx:47:3 ERROR: Expected instr field
 tests\complex-names.mx:48:1 ERROR: Unexpected token '()'
 tests\complex-names.mx:50:2 ERROR: Simple instr field expected
-tests\complex-names.mx:59:1 ERROR: Unexpected token 'ld'
+tests\complex-names.mx:59:1 ERROR: Unexpected token "ld"
 tests\complex-names.mx:60:1 ERROR: Unexpected token '()'
 tests\complex-names.mx:67:20 ERROR: Expected immediate field
 tests\complex-names.mx:68:20 ERROR: Simple immediate field expected
@@ -43,7 +43,7 @@ tests\complex-names.mx:56:1 ERROR: Excessive immediate 'bmeqm 15'
 tests\complex-names.mx:61:1 ERROR: Unknown instruction 'dll'
 tests\complex-names.mx:62:1 ERROR: Unknown condition 'l lvm'
 tests\complex-names.mx:65:1 ERROR: Unknown instruction 'debug 1 234'
-tests\complex-names.mx:66:1 ERROR: Unknown instruction 'debug 01  23 4 5'
-tests\complex-names.mx:69:1 ERROR: Unknown condition 'stst a b'
+tests\complex-names.mx:66:1 ERROR: Unknown instruction 'debug 01  23 "4" 5'
+tests\complex-names.mx:69:1 ERROR: Unknown condition 'stst "a" "b"'