Skip to content

Commit b48533c

Browse files
hossain-khanhossain-khan
committed
[FIXED] Tests for simple-object.json5
1 parent 7da2864 commit b48533c

File tree

1 file changed

+149
-116
lines changed

1 file changed

+149
-116
lines changed

‎lib/src/main/kotlin/io/github/json5/kotlin/JSON5Lexer.kt‎

Lines changed: 149 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ class JSON5Lexer(private val source: String) {
3232
fun nextToken(): Token {
3333
skipWhitespace()
3434
skipComments()
35+
// Fix: skip whitespace and comments in a loop before every token
36+
while (true) {
37+
val before = pos
38+
skipWhitespace()
39+
skipComments()
40+
if (pos == before) break
41+
}
3542

3643
if (currentChar == null) {
3744
return Token.EOFToken(line, column)
@@ -44,136 +51,156 @@ class JSON5Lexer(private val source: String) {
4451
token
4552
}
4653
'"', '\'' -> readString()
47-
'n' -> readNull()
48-
't' -> readTrue()
49-
'f' -> readFalse()
50-
'I' -> readInfinity()
51-
'N' -> readNaN()
52-
'+', '-' -> {
53-
if (peek() == 'I') {
54-
// Handle -Infinity
55-
val sign = currentChar
56-
val startColumn = column
57-
advance()
58-
if (source.substring(pos, minOf(pos + 8, source.length)) == "Infinity") {
59-
repeat(8) { advance() }
60-
return Token.NumericToken(if (sign == '-') Double.NEGATIVE_INFINITY else Double.POSITIVE_INFINITY, line, startColumn)
61-
}
62-
// Not Infinity, revert and continue with normal number parsing
63-
pos -= 1
64-
if (currentChar == '\n') {
65-
line -= 1
66-
column = 1
67-
} else {
68-
column -= 1
69-
}
70-
currentChar = sign
71-
} else if (peek() == 'N') {
72-
// Handle -NaN
73-
val sign = currentChar
74-
val startColumn = column
75-
advance()
76-
if (source.substring(pos, minOf(pos + 3, source.length)) == "NaN") {
77-
repeat(3) { advance() }
78-
return Token.NumericToken(Double.NaN, line, startColumn) // NaN is NaN regardless of sign
79-
}
80-
// Not NaN, revert and continue with normal number parsing
81-
pos -= 1
82-
if (currentChar == '\n') {
83-
line -= 1
84-
column = 1
85-
} else {
86-
column -= 1
54+
// Fix: Only treat 'n', 't', 'f', 'I', 'N' as keywords if they match the full identifier
55+
else -> {
56+
if (isIdentifierStart(currentChar)) {
57+
return readIdentifier()
58+
} else if (currentChar == 'n') {
59+
return readNull()
60+
} else if (currentChar == 't') {
61+
return readTrue()
62+
} else if (currentChar == 'f') {
63+
return readFalse()
64+
} else if (currentChar == 'I') {
65+
return readInfinity()
66+
} else if (currentChar == 'N') {
67+
return readNaN()
68+
} else if (currentChar == '+') {
69+
if (peek() == 'I') {
70+
// Handle +Infinity
71+
val sign = currentChar
72+
val startColumn = column
73+
advance()
74+
if (source.substring(pos, minOf(pos + 8, source.length)) == "Infinity") {
75+
repeat(8) { advance() }
76+
return Token.NumericToken(Double.POSITIVE_INFINITY, line, startColumn)
77+
}
78+
pos -= 1
79+
if (currentChar == '\n') {
80+
line -= 1
81+
column = 1
82+
} else {
83+
column -= 1
84+
}
85+
currentChar = sign
8786
}
88-
currentChar = sign
89-
}
90-
readNumber()
91-
}
92-
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.' -> readNumber()
93-
'\\' -> {
94-
// Handle Unicode escape sequences in identifiers
95-
val startColumn = column
96-
advance() // Skip the backslash
97-
98-
// Now process the escaped character
99-
if (currentChar == 'u') {
100-
advance() // Skip 'u'
101-
val hexDigits = StringBuilder()
102-
repeat(4) {
103-
if (currentChar == null || !currentChar!!.isHexDigit()) {
104-
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
87+
return readNumber()
88+
} else if (currentChar == '-') {
89+
if (peek() == 'I') {
90+
// Handle -Infinity
91+
val sign = currentChar
92+
val startColumn = column
93+
advance()
94+
if (source.substring(pos, minOf(pos + 8, source.length)) == "Infinity") {
95+
repeat(8) { advance() }
96+
return Token.NumericToken(Double.NEGATIVE_INFINITY, line, startColumn)
10597
}
106-
hexDigits.append(currentChar)
98+
pos -= 1
99+
if (currentChar == '\n') {
100+
line -= 1
101+
column = 1
102+
} else {
103+
column -= 1
104+
}
105+
currentChar = sign
106+
} else if (peek() == 'N') {
107+
// Handle -NaN
108+
val sign = currentChar
109+
val startColumn = column
107110
advance()
111+
if (source.substring(pos, minOf(pos + 3, source.length)) == "NaN") {
112+
repeat(3) { advance() }
113+
return Token.NumericToken(Double.NaN, line, startColumn) // NaN is NaN regardless of sign
114+
}
115+
pos -= 1
116+
if (currentChar == '\n') {
117+
line -= 1
118+
column = 1
119+
} else {
120+
column -= 1
121+
}
122+
currentChar = sign
108123
}
124+
return readNumber()
125+
} else if (currentChar in '0'..'9' || currentChar == '.') {
126+
return readNumber()
127+
} else if (currentChar == '\\') {
128+
// Handle Unicode escape sequences in identifiers
129+
val startColumn = column
130+
advance() // Skip the backslash
109131

110-
val char = hexDigits.toString().toInt(16).toChar()
111-
if (!isIdentifierStart(char)) {
112-
throw JSON5Exception.invalidIdentifierChar(line, startColumn)
113-
}
132+
// Now process the escaped character
133+
if (currentChar == 'u') {
134+
advance() // Skip 'u'
135+
val hexDigits = StringBuilder()
136+
repeat(4) {
137+
if (currentChar == null || !currentChar!!.isHexDigit()) {
138+
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
139+
}
140+
hexDigits.append(currentChar)
141+
advance()
142+
}
114143

115-
val buffer = StringBuilder().append(char)
144+
val char = hexDigits.toString().toInt(16).toChar()
145+
if (!isIdentifierStart(char)) {
146+
throw JSON5Exception.invalidIdentifierChar(line, startColumn)
147+
}
116148

117-
// Continue reading the rest of the identifier
118-
while (currentChar != null) {
119-
if (currentChar == '\\') {
120-
val continueColumn = column
121-
advance() // Skip backslash
149+
val buffer = StringBuilder().append(char)
150+
151+
// Continue reading the rest of the identifier
152+
while (currentChar != null) {
153+
if (currentChar == '\\') {
154+
val continueColumn = column
155+
advance() // Skip backslash
156+
157+
if (currentChar == 'u') {
158+
advance() // Skip 'u'
159+
val identHexDigits = StringBuilder()
160+
repeat(4) {
161+
if (currentChar == null || !currentChar!!.isHexDigit()) {
162+
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
163+
}
164+
identHexDigits.append(currentChar)
165+
advance()
166+
}
122167

123-
if (currentChar == 'u') {
124-
advance() // Skip 'u'
125-
val identHexDigits = StringBuilder()
126-
repeat(4) {
127-
if (currentChar == null || !currentChar!!.isHexDigit()) {
128-
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
168+
val continueChar = identHexDigits.toString().toInt(16).toChar()
169+
if (!isIdentifierPart(continueChar)) {
170+
throw JSON5Exception.invalidIdentifierChar(line, continueColumn)
129171
}
130-
identHexDigits.append(currentChar)
131-
advance()
132-
}
133172

134-
val continueChar = identHexDigits.toString().toInt(16).toChar()
135-
if (!isIdentifierPart(continueChar)) {
136-
throw JSON5Exception.invalidIdentifierChar(line, continueColumn)
173+
buffer.append(continueChar)
174+
} else {
175+
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, continueColumn)
137176
}
138-
139-
buffer.append(continueChar)
177+
} else if (isIdentifierPart(currentChar)) {
178+
buffer.append(currentChar)
179+
advance()
140180
} else {
141-
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, continueColumn)
181+
break
142182
}
143-
} else if (isIdentifierPart(currentChar)) {
144-
buffer.append(currentChar)
145-
advance()
146-
} else {
147-
break
148183
}
149-
}
150184

151-
return Token.IdentifierToken(buffer.toString(), line, startColumn)
152-
} else {
153-
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
154-
}
155-
}
156-
'$', '_' -> {
157-
// Handle property names starting with $ or _
158-
readIdentifier()
159-
}
160-
'/' -> {
161-
// Handle incomplete comments
162-
val startColumn = column
163-
val lookAhead = peek()
164-
if (lookAhead == null) {
165-
advance()
166-
throw JSON5Exception.invalidChar('/', line, startColumn)
167-
}
168-
if (lookAhead != '/' && lookAhead != '*') {
169-
advance()
170-
throw JSON5Exception.invalidChar('/', line, startColumn)
171-
}
172-
throw JSON5Exception.invalidChar(lookAhead, line, column + 1)
173-
}
174-
else -> {
175-
if (isIdentifierStart(currentChar)) {
176-
readIdentifier()
185+
return Token.IdentifierToken(buffer.toString(), line, startColumn)
186+
} else {
187+
throw JSON5Exception.invalidChar(currentChar ?: ' ', line, column)
188+
}
189+
} else if (currentChar == '$' || currentChar == '_') {
190+
return readIdentifier()
191+
} else if (currentChar == '/') {
192+
// Handle incomplete comments
193+
val startColumn = column
194+
val lookAhead = peek()
195+
if (lookAhead == null) {
196+
advance()
197+
throw JSON5Exception.invalidChar('/', line, startColumn)
198+
}
199+
if (lookAhead != '/' && lookAhead != '*') {
200+
advance()
201+
throw JSON5Exception.invalidChar('/', line, startColumn)
202+
}
203+
throw JSON5Exception.invalidChar(lookAhead, line, column + 1)
177204
} else {
178205
val c = currentChar ?: ' '
179206
val startColumn = column
@@ -677,7 +704,13 @@ class JSON5Lexer(private val source: String) {
677704
}
678705
}
679706

680-
return Token.IdentifierToken(buffer.toString(), line, startColumn)
707+
val ident = buffer.toString()
708+
return when (ident) {
709+
"true" -> Token.BooleanToken(true, line, startColumn)
710+
"false" -> Token.BooleanToken(false, line, startColumn)
711+
"null" -> Token.NullToken(line, startColumn)
712+
else -> Token.IdentifierToken(ident, line, startColumn)
713+
}
681714
}
682715

683716
private fun readHexEscape(digits: Int): Char {

0 commit comments

Comments
 (0)