Be more careful on invalid tokens (#1092)

Author: dcodeIO

src/diagnosticMessages.generated.ts

@@ -65,6 +65,7 @@ export enum DiagnosticCode {
   An_accessor_cannot_have_type_parameters = 1094,
   A_set_accessor_cannot_have_a_return_type_annotation = 1095,
   Type_parameter_list_cannot_be_empty = 1098,
+  Type_argument_list_cannot_be_empty = 1099,
   A_continue_statement_can_only_be_used_within_an_enclosing_iteration_statement = 1104,
   A_break_statement_can_only_be_used_within_an_enclosing_iteration_or_switch_statement = 1105,
   A_return_statement_can_only_be_used_within_a_function_body = 1108,
@@ -214,6 +215,7 @@ export function diagnosticCodeToString(code: DiagnosticCode): string {
     case 1094: return "An accessor cannot have type parameters.";
     case 1095: return "A 'set' accessor cannot have a return type annotation.";
     case 1098: return "Type parameter list cannot be empty.";
+    case 1099: return "Type argument list cannot be empty.";
     case 1104: return "A 'continue' statement can only be used within an enclosing iteration statement.";
     case 1105: return "A 'break' statement can only be used within an enclosing iteration or switch statement.";
     case 1108: return "A 'return' statement can only be used within a function body.";

src/diagnosticMessages.json

@@ -60,6 +60,7 @@
   "An accessor cannot have type parameters.": 1094,
   "A 'set' accessor cannot have a return type annotation.": 1095,
   "Type parameter list cannot be empty.": 1098,
+  "Type argument list cannot be empty.": 1099,
   "A 'continue' statement can only be used within an enclosing iteration statement.": 1104,
   "A 'break' statement can only be used within an enclosing iteration or switch statement.": 1105,
   "A 'return' statement can only be used within a function body.": 1108,

src/diagnostics.ts

@@ -5,7 +5,8 @@
  *//***/
 
 import {
-  Range
+  Range,
+  Source
 } from "./ast";
 
 import {
@@ -241,7 +242,7 @@ export abstract class DiagnosticEmitter {
   /** Diagnostic messages emitted so far. */
   diagnostics: DiagnosticMessage[];
   /** Diagnostic messages already seen, by range. */
-  private seen: Map<Range,Set<DiagnosticCode>> = new Map();
+  private seen: Map<Source,Map<i32,i32[]>> = new Map();
 
   /** Initializes this diagnostic emitter. */
   protected constructor(diagnostics: DiagnosticMessage[] | null = null) {
@@ -260,17 +261,23 @@ export abstract class DiagnosticEmitter {
   ): void {
     // It is possible that the same diagnostic is emitted twice, for example
     // when compiling generics with different types or when recompiling a loop
-    // because our initial assumptions didn't hold. Deduplicate these.
+    // because our initial assumptions didn't hold. It is even possible to get
+    // multiple instances of the same range during parsing. Deduplicate these.
     if (range) {
       let seen = this.seen;
-      if (seen.has(range)) {
-        let codes = seen.get(range)!;
-        if (codes.has(code)) return;
-        codes.add(code);
+      if (seen.has(range.source)) {
+        let seenInSource = seen.get(range.source)!;
+        if (seenInSource.has(range.start)) {
+          let seenCodesAtPos = seenInSource.get(range.start)!;
+          if (seenCodesAtPos.includes(code)) return;
+          seenCodesAtPos.push(code);
+        } else {
+          seenInSource.set(range.start, [ code ]);
+        }
       } else {
-        let codes = new Set<DiagnosticCode>();
-        codes.add(code);
-        seen.set(range, codes);
+        let seenInSource = new Map();
+        seenInSource.set(range.start, [ code ]);
+        seen.set(range.source, seenInSource);
       }
     }
     var message = DiagnosticMessage.create(code, category, arg0, arg1, arg2);

src/parser.ts

@@ -1043,8 +1043,9 @@ export class Parser extends DiagnosticEmitter {
 
     // at '<': TypeParameter (',' TypeParameter)* '>'
 
-    var typeParameters: TypeParameterNode[] | null = null;
+    var typeParameters = new Array<TypeParameterNode>();
     var seenOptional = false;
+    var start = tn.tokenPos;
     while (!tn.skip(Token.GREATERTHAN)) {
       let typeParameter = this.parseTypeParameter(tn);
       if (!typeParameter) return null;
@@ -1057,8 +1058,7 @@ export class Parser extends DiagnosticEmitter {
         );
         typeParameter.defaultType = null;
       }
-      if (!typeParameters) typeParameters = [ typeParameter ];
-      else typeParameters.push(typeParameter);
+      typeParameters.push(typeParameter);
       if (!tn.skip(Token.COMMA)) {
         if (tn.skip(Token.GREATERTHAN)) {
           break;
@@ -1071,10 +1071,10 @@ export class Parser extends DiagnosticEmitter {
         }
       }
     }
-    if (!(typeParameters && typeParameters.length)) {
+    if (!typeParameters.length) {
       this.error(
         DiagnosticCode.Type_parameter_list_cannot_be_empty,
-        tn.range()
+        tn.range(start, tn.pos)
       ); // recoverable
     }
     return typeParameters;
@@ -3618,7 +3618,8 @@ export class Parser extends DiagnosticEmitter {
 
     var state = tn.mark();
     if (!tn.skip(Token.LESSTHAN)) return null;
-    var typeArguments: TypeNode[] | null = null;
+    var start = tn.tokenPos;
+    var typeArguments = new Array<TypeNode>();
     do {
       if (tn.peek() === Token.GREATERTHAN) {
         break;
@@ -3628,11 +3629,19 @@ export class Parser extends DiagnosticEmitter {
         tn.reset(state);
         return null;
       }
-      if (!typeArguments) typeArguments = [ type ];
-      else typeArguments.push(type);
+      typeArguments.push(type);
     } while (tn.skip(Token.COMMA));
-    if (tn.skip(Token.GREATERTHAN) && tn.skip(Token.OPENPAREN)) {
-      return typeArguments;
+    if (tn.skip(Token.GREATERTHAN)) {
+      let end = tn.pos;
+      if (tn.skip(Token.OPENPAREN)) {
+        if (!typeArguments.length) {
+          this.error(
+            DiagnosticCode.Type_argument_list_cannot_be_empty,
+            tn.range(start, end)
+          );
+        }
+        return typeArguments;
+      }
     }
     tn.reset(state);
     return null;

src/tokenizer.ts

@@ -551,7 +551,11 @@ export class Tokenizer extends DiagnosticEmitter {
 
   next(identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT): Token {
     this.nextToken = -1;
-    return this.token = this.unsafeNext(identifierHandling);
+    var token: Token;
+    do token = this.unsafeNext(identifierHandling);
+    while (token == Token.INVALID);
+    this.token = token;
+    return token;
   }
 
   private unsafeNext(
@@ -962,11 +966,15 @@ export class Tokenizer extends DiagnosticEmitter {
             ++this.pos;
             break;
           }
+          let start = this.pos++;
+          if ( // surrogate pair?
+            (c & 0xFC00) == 0xD800 && this.pos < this.end &&
+            ((text.charCodeAt(this.pos)) & 0xFC00) == 0xDC00
+          ) ++this.pos;
           this.error(
             DiagnosticCode.Invalid_character,
-            this.range(this.pos, this.pos + 1)
+            this.range(start, this.pos)
           );
-          ++this.pos;
           return Token.INVALID;
         }
       }
@@ -984,7 +992,10 @@ export class Tokenizer extends DiagnosticEmitter {
       let posBefore = this.pos;
       let tokenBefore = this.token;
       let tokenPosBefore = this.tokenPos;
-      this.nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
+      let nextToken: Token;
+      do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
+      while (nextToken == Token.INVALID);
+      this.nextToken = nextToken;
       this.nextTokenPos = this.tokenPos;
       if (checkOnNewLine) {
         this.nextTokenOnNewLine = false;
@@ -1017,8 +1028,11 @@ export class Tokenizer extends DiagnosticEmitter {
         break;
       }
     }
-    this.token = this.unsafeNext(identifierHandling, maxCompoundLength);
-    if (this.token == token) {
+    var nextToken: Token;
+    do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
+    while (nextToken == Token.INVALID);
+    if (nextToken == token) {
+      this.token = token;
       this.nextToken = -1;
       return true;
     } else {