context keys: scanner: scan KEY using a regular expression

Author: ulugbekna

src/vs/platform/contextkey/common/scanner.ts

@@ -208,14 +208,6 @@ export class Scanner {
 		return this._isAtEnd() ? '\0' : this._input[this._current];
 	}
 
-	private _peekNext(): string {
-		if (this._current + 1 >= this._input.length) {
-			return '\0';
-		} else {
-			return this._input[this._current + 1];
-		}
-	}
-
 	private _addToken(type: TokenType, captureLexeme: boolean = false) {
 		if (captureLexeme) {
 			const lexeme = this._input.substring(this._start, this._current);
@@ -234,77 +226,22 @@ export class Scanner {
 		this._tokens.push(errToken);
 	}
 
+	private stringRe = /[a-zA-Z0-9_<>\-\./\\:\*\?\+\[\]\^,#@;"%\$\p{L}-]+/uy;
 	private _string() {
-		let peek = this._peek();
-
-		while (this._isStringChar(peek) && !this._isAtEnd()) {
-			this._advance();
-			peek = this._peek();
-		}
-
-		const lexeme = this._input.substring(this._start, this._current);
-
-		const keyword = Scanner._keywords.get(lexeme);
-
-		if (keyword) {
-			this._addToken(keyword);
-		} else {
-			if (lexeme.length > 0) {
+		this.stringRe.lastIndex = this._start;
+		const match = this.stringRe.exec(this._input);
+		if (match) {
+			this._current = this._start + match[0].length;
+			const lexeme = this._input.substring(this._start, this._current);
+			const keyword = Scanner._keywords.get(lexeme);
+			if (keyword) {
+				this._addToken(keyword);
+			} else {
 				this._tokens.push({ type: TokenType.Str, lexeme, offset: this._start });
 			}
 		}
 	}
 
-	private _isStringChar(peek: string) {
-		if (this._isAlphaNumeric(peek)) { return true; }
-		if (this._isWhitespace(peek)) { return false; }
-
-		switch (peek) {
-			case ')':
-			case '=':
-			case '!':
-			case '&':
-			case '|':
-			case '~':
-				return false;
-			case '<':
-			case '>':
-				return (this._peekNext() === '=') ? false : true; // so that we support `foo<=1` as `foo <= 1`, but also support `vim.use<C-r>` as a single KEY
-			case '_':
-			case '-':
-			case '.':
-			case '/':
-			case '\\':
-			case ':':
-			case '*':
-			case '?': // do we have to?
-			case '%':
-			case '+':
-			case '[':
-			case ']':
-			case '^':
-			case ',':
-			case '#':
-			case '@':
-			case ';':
-			case '"':
-				return true;
-			default:
-				if (peek.charCodeAt(0) > 127) { // handle unicode, eg Chinese hieroglyphs used in extensions
-					return true;
-				}
-				return false;
-		}
-	}
-
-	private _isAlphaNumeric(ch: string) {
-		return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9');
-	}
-
-	private _isWhitespace(ch: string) {
-		return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r' || ch === /* &nbsp */ '\u00A0';
-	}
-
 	// captures the lexeme without the leading and trailing '
 	private _quotedString() {
 		while (this._peek() !== `'` && !this._isAtEnd()) { // TODO@ulugbekna: add support for escaping ' ?

src/vs/platform/contextkey/test/common/parser.test.ts

@@ -169,7 +169,7 @@ suite('Context Key Scanner', () => {
 
 		test('vim<c-r> == 1 && vim<2<=3', () => {
 			const input = 'vim<c-r> == 1 && vim<2<=3';
-			assert.deepStrictEqual(parseToStr(input), "vim<c-r> == '1' && vim<2 <= 3"); // FIXME
+			assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token '=' at offset 23. Did you mean '==' or '=~'?\n"); // FIXME
 		});
 
 		test(`foo && 'bar`, () => {

src/vs/platform/contextkey/test/common/scanner.test.ts

@@ -106,6 +106,11 @@ suite('Context Key Scanner', () => {
 			assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "foo", offset: 0 }, { type: "=~", offset: 4 }, { type: "RegexStr", lexeme: "/zee/gm", offset: 7 }, { type: "EOF", offset: 14 }]));
 		});
 
+		test('foo in barrr  ', () => {
+			const input = 'foo in barrr  ';
+			assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "foo", offset: 0 }, { type: "in", offset: 4 }, { type: "Str", lexeme: "barrr", offset: 7 }, { type: "EOF", offset: 14 }]));
+		});
+
 		test('editorLangId in testely.supportedLangIds && resourceFilename =~ /^.+(.test.(\w+))$/gm', () => {
 			const input = 'editorLangId in testely.supportedLangIds && resourceFilename =~ /^.+(.test.(\w+))$/gm';
 			assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "editorLangId", offset: 0 }, { type: "in", offset: 13 }, { type: "Str", lexeme: "testely.supportedLangIds", offset: 16 }, { type: "&&", offset: 41 }, { type: "Str", lexeme: "resourceFilename", offset: 44 }, { type: "=~", offset: 61 }, { type: "RegexStr", lexeme: "/^.+(.test.(w+))$/gm", offset: 64 }, { type: "EOF", offset: 84 }]));
@@ -193,7 +198,7 @@ suite('Context Key Scanner', () => {
 
 		test('vim<c-r>==1 && vim<2<=3', () => {
 			const input = 'vim<c-r>==1 && vim<2<=3';
-			assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "vim<c-r", offset: 0 }, { type: ">=", offset: 7 }, { type: "ErrorToken", offset: 9, lexeme: "=" }, { type: "&&", offset: 12 }, { type: "Str", lexeme: "vim<2", offset: 15 }, { type: "<=", offset: 20 }, { type: "Str", lexeme: "3", offset: 22 }, { type: "EOF", offset: 23 }]));
+			assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "vim<c-r>", offset: 0 }, { type: "==", offset: 8 }, { type: "Str", lexeme: "1", offset: 10 }, { type: "&&", offset: 12 }, { type: "Str", lexeme: "vim<2<", offset: 15 }, { type: "ErrorToken", offset: 21, lexeme: "=" }, { type: "EOF", offset: 23 }]));
 		});
 	});
 });