Skip to content

Commit 7d3f063

Browse files
ruby0x1ruby0x1
committed
Merge branch 'next-token' into main
2 parents 4687300 + 3c475f0 commit 7d3f063

File tree

2 files changed

+196
-19
lines changed

2 files changed

+196
-19
lines changed

‎src/vm/wren_compiler.c‎

Lines changed: 45 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ typedef struct
173173
// The 1-based line number of [currentChar].
174174
int currentLine;
175175

176+
// The upcoming token.
177+
Token next;
178+
176179
// The most recently lexed token.
177180
Token current;
178181

@@ -639,13 +642,13 @@ static bool matchChar(Parser* parser, char c)
639642
// range.
640643
static void makeToken(Parser* parser, TokenType type)
641644
{
642-
parser->current.type = type;
643-
parser->current.start = parser->tokenStart;
644-
parser->current.length = (int)(parser->currentChar - parser->tokenStart);
645-
parser->current.line = parser->currentLine;
645+
parser->next.type = type;
646+
parser->next.start = parser->tokenStart;
647+
parser->next.length = (int)(parser->currentChar - parser->tokenStart);
648+
parser->next.line = parser->currentLine;
646649

647650
// Make line tokens appear on the line containing the "\n".
648-
if (type == TOKEN_LINE) parser->current.line--;
651+
if (type == TOKEN_LINE) parser->next.line--;
649652
}
650653

651654
// If the current character is [c], then consumes it and makes a token of type
@@ -719,17 +722,17 @@ static void makeNumber(Parser* parser, bool isHex)
719722

720723
if (isHex)
721724
{
722-
parser->current.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
725+
parser->next.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
723726
}
724727
else
725728
{
726-
parser->current.value = NUM_VAL(strtod(parser->tokenStart, NULL));
729+
parser->next.value = NUM_VAL(strtod(parser->tokenStart, NULL));
727730
}
728731

729732
if (errno == ERANGE)
730733
{
731734
lexError(parser, "Number literal was too large (%d).", sizeof(long int));
732-
parser->current.value = NUM_VAL(0);
735+
parser->next.value = NUM_VAL(0);
733736
}
734737

735738
// We don't check that the entire token is consumed after calling strtoll()
@@ -921,21 +924,23 @@ static void readString(Parser* parser)
921924
}
922925
}
923926

924-
parser->current.value = wrenNewStringLength(parser->vm,
927+
parser->next.value = wrenNewStringLength(parser->vm,
925928
(char*)string.data, string.count);
926929

927930
wrenByteBufferClear(parser->vm, &string);
928931
makeToken(parser, type);
929932
}
930933

931-
// Lex the next token and store it in [parser.current].
934+
// Lex the next token and store it in [parser.next].
932935
static void nextToken(Parser* parser)
933936
{
934937
parser->previous = parser->current;
938+
parser->current = parser->next;
935939

936940
// If we are out of tokens, don't try to tokenize any more. We *do* still
937941
// copy the TOKEN_EOF to previous so that code that expects it to be consumed
938942
// will still work.
943+
if (parser->next.type == TOKEN_EOF) return;
939944
if (parser->current.type == TOKEN_EOF) return;
940945

941946
while (peekChar(parser) != '\0')
@@ -1094,8 +1099,8 @@ static void nextToken(Parser* parser)
10941099
// even though the source code and console output are UTF-8.
10951100
lexError(parser, "Invalid byte 0x%x.", (uint8_t)c);
10961101
}
1097-
parser->current.type = TOKEN_ERROR;
1098-
parser->current.length = 0;
1102+
parser->next.type = TOKEN_ERROR;
1103+
parser->next.length = 0;
10991104
}
11001105
return;
11011106
}
@@ -1114,6 +1119,12 @@ static TokenType peek(Compiler* compiler)
11141119
return compiler->parser->current.type;
11151120
}
11161121

1122+
// Returns the type of the current token.
1123+
static TokenType peekNext(Compiler* compiler)
1124+
{
1125+
return compiler->parser->next.type;
1126+
}
1127+
11171128
// Consumes the current token if its type is [expected]. Returns true if a
11181129
// token was consumed.
11191130
static bool match(Compiler* compiler, TokenType expected)
@@ -1162,6 +1173,12 @@ static void consumeLine(Compiler* compiler, const char* errorMessage)
11621173
ignoreNewlines(compiler);
11631174
}
11641175

1176+
static void allowLineBeforeDot(Compiler* compiler) {
1177+
if (peek(compiler) == TOKEN_LINE && peekNext(compiler) == TOKEN_DOT) {
1178+
nextToken(compiler->parser);
1179+
}
1180+
}
1181+
11651182
// Variables and scopes --------------------------------------------------------
11661183

11671184
// Emits one single-byte argument. Returns its index.
@@ -1949,6 +1966,7 @@ static void namedCall(Compiler* compiler, bool canAssign, Code instruction)
19491966
else
19501967
{
19511968
methodCall(compiler, instruction, &signature);
1969+
allowLineBeforeDot(compiler);
19521970
}
19531971
}
19541972

@@ -2145,6 +2163,8 @@ static void field(Compiler* compiler, bool canAssign)
21452163
loadThis(compiler);
21462164
emitByteArg(compiler, isLoad ? CODE_LOAD_FIELD : CODE_STORE_FIELD, field);
21472165
}
2166+
2167+
allowLineBeforeDot(compiler);
21482168
}
21492169

21502170
// Compiles a read or assignment to [variable].
@@ -2176,6 +2196,8 @@ static void bareName(Compiler* compiler, bool canAssign, Variable variable)
21762196

21772197
// Emit the load instruction.
21782198
loadVariable(compiler, variable);
2199+
2200+
allowLineBeforeDot(compiler);
21792201
}
21802202

21812203
static void staticField(Compiler* compiler, bool canAssign)
@@ -2360,6 +2382,8 @@ static void subscript(Compiler* compiler, bool canAssign)
23602382
finishArgumentList(compiler, &signature);
23612383
consume(compiler, TOKEN_RIGHT_BRACKET, "Expect ']' after arguments.");
23622384

2385+
allowLineBeforeDot(compiler);
2386+
23632387
if (canAssign && match(compiler, TOKEN_EQ))
23642388
{
23652389
signature.type = SIG_SUBSCRIPT_SETTER;
@@ -3488,19 +3512,21 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
34883512
parser.numParens = 0;
34893513

34903514
// Zero-init the current token. This will get copied to previous when
3491-
// advance() is called below.
3492-
parser.current.type = TOKEN_ERROR;
3493-
parser.current.start = source;
3494-
parser.current.length = 0;
3495-
parser.current.line = 0;
3496-
parser.current.value = UNDEFINED_VAL;
3515+
// nextToken() is called below.
3516+
parser.next.type = TOKEN_ERROR;
3517+
parser.next.start = source;
3518+
parser.next.length = 0;
3519+
parser.next.line = 0;
3520+
parser.next.value = UNDEFINED_VAL;
34973521

34983522
// Ignore leading newlines.
34993523
parser.skipNewlines = true;
35003524
parser.printErrors = printErrors;
35013525
parser.hasError = false;
35023526

3503-
// Read the first token.
3527+
// Read the first token into next
3528+
nextToken(&parser);
3529+
// Copy next -> current
35043530
nextToken(&parser);
35053531

35063532
int numExistingVariables = module->variables.count;

‎test/language/chained_newline.wren‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
class Test {
2+
construct new() {}
3+
test0() {
4+
System.print("test0")
5+
return this
6+
}
7+
test1() {
8+
System.print("test1")
9+
return this
10+
}
11+
test2() {
12+
System.print("test2")
13+
return this
14+
}
15+
16+
[index] {
17+
System.print("testSubscript")
18+
return this
19+
}
20+
}
21+
22+
class Tester {
23+
construct new() {
24+
25+
var test = _test = Test.new()
26+
27+
//test local access
28+
29+
test.
30+
test0(). // expect: test0
31+
test1(). // expect: test1
32+
test2() // expect: test2
33+
34+
test
35+
.test0() // expect: test0
36+
.test1() // expect: test1
37+
.test2() // expect: test2
38+
39+
test
40+
.test0() // expect: test0
41+
.test1(). // expect: test1
42+
test2() // expect: test2
43+
44+
test[0] // expect: testSubscript
45+
.test0() // expect: test0
46+
47+
test[0]. // expect: testSubscript
48+
test0() // expect: test0
49+
50+
//test field access
51+
52+
_test.
53+
test0(). // expect: test0
54+
test1(). // expect: test1
55+
test2() // expect: test2
56+
57+
_test
58+
.test0() // expect: test0
59+
.test1() // expect: test1
60+
.test2() // expect: test2
61+
62+
_test
63+
.test0(). // expect: test0
64+
test1(). // expect: test1
65+
test2() // expect: test2
66+
67+
_test[0] // expect: testSubscript
68+
.test0() // expect: test0
69+
70+
_test[0]. // expect: testSubscript
71+
test0() // expect: test0
72+
73+
}
74+
75+
getter { _test }
76+
method() { _test }
77+
78+
} //Tester
79+
80+
//access via methods/getter
81+
82+
var external = Tester.new()
83+
84+
external.getter.
85+
test0(). // expect: test0
86+
test1(). // expect: test1
87+
test2() // expect: test2
88+
89+
external.getter
90+
.test0() // expect: test0
91+
.test1() // expect: test1
92+
.test2() // expect: test2
93+
94+
external.getter.
95+
test0() // expect: test0
96+
.test1() // expect: test1
97+
.test2() // expect: test2
98+
99+
external.getter[0]. // expect: testSubscript
100+
test0() // expect: test0
101+
102+
external.getter[0] // expect: testSubscript
103+
.test0() // expect: test0
104+
105+
external.method().
106+
test0(). // expect: test0
107+
test1(). // expect: test1
108+
test2() // expect: test2
109+
110+
external.method()
111+
.test0() // expect: test0
112+
.test1() // expect: test1
113+
.test2() // expect: test2
114+
115+
external.method().
116+
test0() // expect: test0
117+
.test1(). // expect: test1
118+
test2() // expect: test2
119+
120+
external.method()[0]. // expect: testSubscript
121+
test0() // expect: test0
122+
123+
external.method()[0] // expect: testSubscript
124+
.test0() // expect: test0
125+
126+
127+
//regular access in module scope
128+
129+
var other = Test.new()
130+
131+
other.
132+
test0(). // expect: test0
133+
test1(). // expect: test1
134+
test2() // expect: test2
135+
136+
other
137+
.test0() // expect: test0
138+
.test1() // expect: test1
139+
.test2() // expect: test2
140+
141+
other
142+
.test0(). // expect: test0
143+
test1() // expect: test1
144+
.test2() // expect: test2
145+
146+
147+
other[0] // expect: testSubscript
148+
.test0() // expect: test0
149+
150+
other[0]. // expect: testSubscript
151+
test0() // expect: test0

0 commit comments

Comments
 (0)