Skip to content

Commit f866e5d

Browse files
ulugbeknaulugbekna
committed
context keys: parser: it can now reconstruct regexp even if uses unescaped slashes
1 parent 72f33d5 commit f866e5d

File tree

3 files changed

+103
-12
lines changed

3 files changed

+103
-12
lines changed

‎src/vs/platform/contextkey/common/contextkey.ts

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6+
import { CharCode } from 'vs/base/common/charCode';
67
import { Event } from 'vs/base/common/event';
78
import { isChrome, isEdge, isFirefox, isLinux, isMacintosh, isSafari, isWeb, isWindows } from 'vs/base/common/platform';
89
import { isFalsyOrWhitespace } from 'vs/base/common/strings';
@@ -356,9 +357,7 @@ export class Parser {
356357
}
357358

358359
this._tokens = this._scanner.reset(input).scan();
359-
if (this._scanner.errorTokens.length > 0) {
360-
return undefined;
361-
}
360+
// @ulugbekna: we do not stop parsing if there are lexing errors to be able to reconstruct regexes with unescaped slashes
362361

363362
this._current = 0;
364363
this._parsingErrors = [];
@@ -451,14 +450,55 @@ export class Parser {
451450
// =~ regex
452451
if (this._matchOne(TokenType.RegexOp)) {
453452

453+
// @ulugbekna: we need to reconstruct the regex from the tokens because some extensions use unescaped slashes in regexes
454454
const expr = this._peek();
455455
switch (expr.type) {
456-
case TokenType.RegexStr: {
457-
const regexLexeme = expr.lexeme; // /REGEX/ or /REGEX/FLAGS
456+
case TokenType.RegexStr:
457+
case TokenType.Error: { // also handle an ErrorToken in case of smth such as /(/file)/
458+
const lexemeReconstruction = [expr.lexeme]; // /REGEX/ or /REGEX/FLAGS
458459
this._advance();
460+
461+
let followingToken = this._peek();
462+
let parenBalance = 0;
463+
for (let i = 0; i < expr.lexeme.length; i++) {
464+
if (expr.lexeme.charCodeAt(i) === CharCode.OpenParen) {
465+
parenBalance++;
466+
} else if (expr.lexeme.charCodeAt(i) === CharCode.CloseParen) {
467+
parenBalance--;
468+
}
469+
}
470+
471+
while (!this._isAtEnd() && followingToken.type !== TokenType.And && followingToken.type !== TokenType.Or) {
472+
switch (followingToken.type) {
473+
case TokenType.LParen:
474+
parenBalance++;
475+
break;
476+
case TokenType.RParen:
477+
parenBalance--;
478+
break;
479+
case TokenType.RegexStr:
480+
case TokenType.QuotedStr:
481+
for (let i = 0; i < followingToken.lexeme.length; i++) {
482+
if (followingToken.lexeme.charCodeAt(i) === CharCode.OpenParen) {
483+
parenBalance++;
484+
} else if (expr.lexeme.charCodeAt(i) === CharCode.CloseParen) {
485+
parenBalance--;
486+
}
487+
}
488+
}
489+
if (parenBalance < 0) {
490+
break;
491+
}
492+
lexemeReconstruction.push(Scanner.getLexeme(followingToken));
493+
this._advance();
494+
followingToken = this._peek();
495+
}
496+
497+
const regexLexeme = lexemeReconstruction.join('');
459498
const closingSlashIndex = regexLexeme.lastIndexOf('/');
460499
const flags = closingSlashIndex === regexLexeme.length - 1 ? undefined : regexLexeme.substring(closingSlashIndex + 1);
461-
return ContextKeyExpr.regex(key, new RegExp(regexLexeme.substring(1, closingSlashIndex), flags));
500+
const regexp = new RegExp(regexLexeme.substring(1, closingSlashIndex), flags);
501+
return ContextKeyExpr.regex(key, regexp);
462502
}
463503

464504
case TokenType.QuotedStr: {
@@ -635,9 +675,6 @@ export class Parser {
635675
return this._peek().type === type;
636676
}
637677

638-
/*
639-
Careful: the function doesn't check array bounds.
640-
*/
641678
private _peek() {
642679
return this._tokens[this._current];
643680
}

‎src/vs/platform/contextkey/test/common/parser.test.ts

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,37 @@ suite('Context Key Scanner', () => {
150150

151151
});
152152

153+
suite('regex', () => {
154+
155+
test(`resource =~ //foo/(barr|door/(Foo-Bar%20Templates|Soo%20Looo)|Web%20Site%Jjj%20Llll)(/.*)*$/`, () => {
156+
const input = `resource =~ //foo/(barr|door/(Foo-Bar%20Templates|Soo%20Looo)|Web%20Site%Jjj%20Llll)(/.*)*$/`;
157+
assert.deepStrictEqual(parseToStr(input), "resource =~ /\\/FileCabinet\\/(SuiteScripts|Templates\\/(E-mail%20Templates|Marketing%20Templates)|Web%20Site%20Hosting%20Files)(\\/.*)*$/");
158+
});
159+
160+
test(`resource =~ /((/scratch/(?!update)(.*)/)|((/src/).*/)).*$/`, () => {
161+
const input = `resource =~ /((/scratch/(?!update)(.*)/)|((/src/).*/)).*$/`;
162+
assert.deepStrictEqual(parseToStr(input), "resource =~ /((\\/scratch\\/(?!update)(.*)\\/)|((\\/src\\/).*\\/)).*$/");
163+
});
164+
165+
test(`FIXME resourcePath =~ //foo/barr// || resourcePath =~ //view/(foo|frontend|base)/(foo|barr)// && resourceExtname in fooBar`, () => {
166+
const input = `resourcePath =~ //foo/barr// || resourcePath =~ //view/(foo|frontend|base)/(foo|barr)// && resourceExtname in fooBar`;
167+
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token '|' at offset 59. Did you mean '||'?\nUnexpected token '|' at offset 68. Did you mean '||'?\nUnexpected token '/ && resourceExtname in fooBar' at offset 86\n\n --- \nParsing errors:\n\nUnexpected error: SyntaxError: Invalid flags supplied to RegExp constructor ' && resourceExtname in fooBar' for token EOF at offset 116.\n");
168+
});
169+
170+
test(`resourcePath =~ /\.md(\.yml|\.txt)*$/gim`, () => {
171+
const input = `resourcePath =~ /\.md(\.yml|\.txt)*$/gim`;
172+
assert.deepStrictEqual(parseToStr(input), "resourcePath =~ /.md(.yml|.txt)*$/gim");
173+
});
174+
175+
});
176+
153177
suite('error handling', () => {
154178

179+
test(`/foo`, () => {
180+
const input = `/foo`;
181+
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token '/foo' at offset 0\n\n --- \nParsing errors:\n\nExpected 'true', 'false', '(', KEY, KEY '=~' regex, KEY [ ('==' | '!=' | '<' | '<=' | '>' | '>=' | 'in' | 'not' 'in') value ] but got '/foo' at offset 0.\n");
182+
});
183+
155184
test(`!b == 'true'`, () => {
156185
const input = `!b == 'true'`;
157186
assert.deepStrictEqual(parseToStr(input), "Parsing errors:\n\nUnexpected '==' at offset 3.\n");
@@ -164,17 +193,17 @@ suite('Context Key Scanner', () => {
164193

165194
test(`view =~ '/(servers)/' && viewItem =~ /^(Starting|Started|Debugging|Stopping|Stopped|Unknown)/'`, () => {
166195
const input = `view =~ '/(servers)/' && viewItem =~ /^(Starting|Started|Debugging|Stopping|Stopped|Unknown)/'`;
167-
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token ''' at offset 93\n");
196+
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token ''' at offset 93\n\n --- \nParsing errors:\n\nUnexpected error: SyntaxError: Invalid flags supplied to RegExp constructor ''' for token EOF at offset 94.\n");
168197
});
169198

170199
test('vim<c-r> == 1 && vim<2<=3', () => {
171200
const input = 'vim<c-r> == 1 && vim<2<=3';
172-
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token '=' at offset 23. Did you mean '==' or '=~'?\n"); // FIXME
201+
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token '=' at offset 23. Did you mean '==' or '=~'?\n\n --- \nParsing errors:\n\nUnexpected '=' at offset 23.\n"); // FIXME
173202
});
174203

175204
test(`foo && 'bar`, () => {
176205
const input = `foo && 'bar`;
177-
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token ''bar' at offset 7. Did you forget to close the string?\n");
206+
assert.deepStrictEqual(parseToStr(input), "Lexing errors:\n\nUnexpected token ''bar' at offset 7. Did you forget to close the string?\n\n --- \nParsing errors:\n\nExpected 'true', 'false', '(', KEY, KEY '=~' regex, KEY [ ('==' | '!=' | '<' | '<=' | '>' | '>=' | 'in' | 'not' 'in') value ] but got ''bar' at offset 7.\n");
178207
});
179208

180209
/*

‎src/vs/platform/contextkey/test/common/scanner.test.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,11 @@ suite('Context Key Scanner', () => {
110110
assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "foo", offset: 0 }, { type: "in", offset: 4 }, { type: "Str", lexeme: "barrr", offset: 7 }, { type: "EOF", offset: 14 }]));
111111
});
112112

113+
test(`resource =~ //FileCabinet/(SuiteScripts|Templates/(E-mail%20Templates|Marketing%20Templates)|Web%20Site%20Hosting%20Files)(/.*)*$/`, () => {
114+
const input = `resource =~ //FileCabinet/(SuiteScripts|Templates/(E-mail%20Templates|Marketing%20Templates)|Web%20Site%20Hosting%20Files)(/.*)*$/`;
115+
assert.deepStrictEqual(scan(input), ([{ type: "Str", offset: 0, lexeme: "resource" }, { type: "=~", offset: 9 }, { type: "RegexStr", offset: 12, lexeme: "//" }, { type: "Str", offset: 14, lexeme: "FileCabinet/" }, { type: "(", offset: 26 }, { type: "Str", offset: 27, lexeme: "SuiteScripts" }, { type: "ErrorToken", offset: 39, lexeme: "|" }, { type: "Str", offset: 40, lexeme: "Templates/" }, { type: "(", offset: 50 }, { type: "Str", offset: 51, lexeme: "E-mail%20Templates" }, { type: "ErrorToken", offset: 69, lexeme: "|" }, { type: "Str", offset: 70, lexeme: "Marketing%20Templates" }, { type: ")", offset: 91 }, { type: "ErrorToken", offset: 92, lexeme: "|" }, { type: "Str", offset: 93, lexeme: "Web%20Site%20Hosting%20Files" }, { type: ")", offset: 121 }, { type: "(", offset: 122 }, { type: "RegexStr", offset: 123, lexeme: "/.*)*$/" }, { type: "EOF", offset: 130 }]));
116+
});
117+
113118
test('editorLangId in testely.supportedLangIds && resourceFilename =~ /^.+(.test.(\w+))$/gm', () => {
114119
const input = 'editorLangId in testely.supportedLangIds && resourceFilename =~ /^.+(.test.(\w+))$/gm';
115120
assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "editorLangId", offset: 0 }, { type: "in", offset: 13 }, { type: "Str", lexeme: "testely.supportedLangIds", offset: 16 }, { type: "&&", offset: 41 }, { type: "Str", lexeme: "resourceFilename", offset: 44 }, { type: "=~", offset: 61 }, { type: "RegexStr", lexeme: "/^.+(.test.(w+))$/gm", offset: 64 }, { type: "EOF", offset: 84 }]));
@@ -169,6 +174,11 @@ suite('Context Key Scanner', () => {
169174
const input = `view =~ '/(servers)/' && viewItem =~ '/^(Starting|Started|Debugging|Stopping|Stopped)/'`;
170175
assert.deepStrictEqual(scan(input), ([{ type: "Str", lexeme: "view", offset: 0 }, { type: "=~", offset: 5 }, { type: "QuotedStr", lexeme: "/(servers)/", offset: 9 }, { type: "&&", offset: 22 }, { type: "Str", lexeme: "viewItem", offset: 25 }, { type: "=~", offset: 34 }, { type: "QuotedStr", lexeme: "/^(Starting|Started|Debugging|Stopping|Stopped)/", offset: 38 }, { type: "EOF", offset: 87 }]));
171176
});
177+
178+
test(`resourcePath =~ /\.md(\.yml|\.txt)*$/gim`, () => {
179+
const input = `resourcePath =~ /\.md(\.yml|\.txt)*$/gim`;
180+
assert.deepStrictEqual(scan(input), ([{ type: "Str", offset: 0, lexeme: "resourcePath" }, { type: "=~", offset: 13 }, { type: "RegexStr", offset: 16, lexeme: "/.md(.yml|.txt)*$/gim" }, { type: "EOF", offset: 37 }]));
181+
});
172182
});
173183

174184
suite('handling lexical errors', () => {
@@ -204,5 +214,20 @@ suite('Context Key Scanner', () => {
204214
const input = `foo|bar`;
205215
assert.deepStrictEqual(scan(input), ([{ type: "Str", offset: 0, lexeme: "foo" }, { type: "ErrorToken", offset: 3, lexeme: "|" }, { type: "Str", offset: 4, lexeme: "bar" }, { type: "EOF", offset: 7 }]));
206216
});
217+
218+
test(`resource =~ //foo/(barr|door/(Foo-Bar%20Templates|Soo%20Looo)|Web%20Site%Jjj%20Llll)(/.*)*$/`, () => {
219+
const input = `resource =~ //foo/(barr|door/(Foo-Bar%20Templates|Soo%20Looo)|Web%20Site%Jjj%20Llll)(/.*)*$/`;
220+
assert.deepStrictEqual(scan(input), ([{ type: "Str", offset: 0, lexeme: "resource" }, { type: "=~", offset: 9 }, { type: "RegexStr", offset: 12, lexeme: "//" }, { type: "Str", offset: 14, lexeme: "foo/" }, { type: "(", offset: 18 }, { type: "Str", offset: 19, lexeme: "barr" }, { type: "ErrorToken", offset: 23, lexeme: "|" }, { type: "Str", offset: 24, lexeme: "door/" }, { type: "(", offset: 29 }, { type: "Str", offset: 30, lexeme: "Foo-Bar%20Templates" }, { type: "ErrorToken", offset: 49, lexeme: "|" }, { type: "Str", offset: 50, lexeme: "Soo%20Looo" }, { type: ")", offset: 60 }, { type: "ErrorToken", offset: 61, lexeme: "|" }, { type: "Str", offset: 62, lexeme: "Web%20Site%Jjj%20Llll" }, { type: ")", offset: 83 }, { type: "(", offset: 84 }, { type: "RegexStr", offset: 85, lexeme: "/.*)*$/" }, { type: "EOF", offset: 92 }]));
221+
});
222+
223+
test(`/((/foo/(?!bar)(.*)/)|((/src/).*/)).*$/`, () => {
224+
const input = `/((/foo/(?!bar)(.*)/)|((/src/).*/)).*$/`;
225+
assert.deepStrictEqual(scan(input), ([{ type: "RegexStr", offset: 0, lexeme: "/((/" }, { type: "Str", offset: 4, lexeme: "foo/" }, { type: "(", offset: 8 }, { type: "Str", offset: 9, lexeme: "?" }, { type: "!", offset: 10 }, { type: "Str", offset: 11, lexeme: "bar" }, { type: ")", offset: 14 }, { type: "(", offset: 15 }, { type: "Str", offset: 16, lexeme: ".*" }, { type: ")", offset: 18 }, { type: "RegexStr", offset: 19, lexeme: "/)|((/s" }, { type: "Str", offset: 26, lexeme: "rc/" }, { type: ")", offset: 29 }, { type: "Str", offset: 30, lexeme: ".*/" }, { type: ")", offset: 33 }, { type: ")", offset: 34 }, { type: "Str", offset: 35, lexeme: ".*$/" }, { type: "EOF", offset: 39 }]));
226+
});
227+
228+
test(`resourcePath =~ //foo/barr// || resourcePath =~ //view/(jarrr|doooor|bees)/(web|templates)// && resourceExtname in foo.Bar`, () => {
229+
const input = `resourcePath =~ //foo/barr// || resourcePath =~ //view/(jarrr|doooor|bees)/(web|templates)// && resourceExtname in foo.Bar`;
230+
assert.deepStrictEqual(scan(input), ([{ type: "Str", offset: 0, lexeme: "resourcePath" }, { type: "=~", offset: 13 }, { type: "RegexStr", offset: 16, lexeme: "//" }, { type: "Str", offset: 18, lexeme: "foo/barr//" }, { type: "||", offset: 29 }, { type: "Str", offset: 32, lexeme: "resourcePath" }, { type: "=~", offset: 45 }, { type: "RegexStr", offset: 48, lexeme: "//" }, { type: "Str", offset: 50, lexeme: "view/" }, { type: "(", offset: 55 }, { type: "Str", offset: 56, lexeme: "jarrr" }, { type: "ErrorToken", offset: 61, lexeme: "|" }, { type: "Str", offset: 62, lexeme: "doooor" }, { type: "ErrorToken", offset: 68, lexeme: "|" }, { type: "Str", offset: 69, lexeme: "bees" }, { type: ")", offset: 73 }, { type: "RegexStr", offset: 74, lexeme: "/(web|templates)/" }, { type: "ErrorToken", offset: 91, lexeme: "/ && resourceExtname in foo.Bar" }, { type: "EOF", offset: 122 }]));
231+
});
207232
});
208233
});

0 commit comments

Comments
 (0)