Add support for multi-line parentheses

Author: Fidget-Spinner

src/errors.ts

@@ -114,6 +114,17 @@ export namespace TokenizerErrors {
             this.name = "ForbiddenOperatorError";
         }
     }
+
+    export class NonMatchingParenthesesError extends BaseTokenizerError {
+        constructor(line: number, col: number, source: string, current: number) {
+            let msg = getFullLine(source, current-1) + "\n";
+            let hint = `${col > 1 ? '~' : ''}^~ Non-matching closing parentheses.`;
+            // The extra `~` character takes up some space.
+            hint = hint.padStart(hint.length + col - MAGIC_OFFSET - (col > 1 ? 1 : 0), " ");
+            super(msg + hint, line, col);
+            this.name = "NonMatchingParenthesesError";
+        }
+    }
 }
 
 export namespace ParserErrors {

src/tests/regression.test.ts

@@ -15,6 +15,18 @@ def foo():
 print("hi")
         
 print("world")
+`;
+        toEstreeAST(text);
+    })
+    test('Issue #3', () => {
+        const text = `
+def foo(
+    a,
+    b
+):
+    pass
+
+    pass
 `;
         toEstreeAST(text);
     })

src/tokenizer.ts

@@ -88,6 +88,7 @@ const specialIdentifiers = new Map([
 
 export const SPECIAL_IDENTIFIER_TOKENS = Array.from(specialIdentifiers.values());
 
+
 export class Tokenizer {
     private readonly source: string;
     private readonly tokens: Token[];
@@ -98,6 +99,7 @@ export class Tokenizer {
     private readonly indentStack: number[];
     private specialIdentifiers: Map<string, TokenType>;
     private forbiddenIdentifiers: Map<string, TokenType>;
+    private parenthesesLevel: number;
 
     // forbiddenOperators: Set<TokenType>;
     constructor(source: string) {
@@ -139,6 +141,7 @@ export class Tokenizer {
         //     TokenType.DOUBLESTAREQUAL,
         //     TokenType.DOUBLESLASHEQUAL,
         // ])
+        this.parenthesesLevel = 0;
     }
 
     private isAtEnd() {
@@ -273,6 +276,11 @@ export class Tokenizer {
                     break;
                 }
             case '\n':
+                if (this.parenthesesLevel > 0) {
+                    this.line += 1;
+                    this.col = 0;
+                    break;
+                }
                 this.addToken(TokenType.NEWLINE);
                 this.line += 1;
                 this.col = 0;
@@ -362,9 +370,14 @@ export class Tokenizer {
             //// Everything else
             case '(':
                 this.addToken(TokenType.LPAR);
+                this.parenthesesLevel++;
                 break;
             case ')':
                 this.addToken(TokenType.RPAR);
+                if (this.parenthesesLevel === 0) {
+                    throw new TokenizerErrors.NonMatchingParenthesesError(this.line, this.col, this.source, this.current);
+                }
+                this.parenthesesLevel--;
                 break;
             case ',':
                 this.addToken(TokenType.COMMA);

test.py

@@ -1,4 +1,6 @@
-def foo():
+def foo(
+    a, b
+):
     pass
 
     pass