Skip to content

Commit 2be4ef1

Browse files
DustinCampbellDustinCampbell
authored
Merge pull request #742 from ivanz/ivan-syntax-highlighting-tests
Added test framework and a couple of tests for for syntax highlighting
2 parents 3e8299f + 358aea8 commit 2be4ef1

File tree

11 files changed

+875
-12
lines changed

11 files changed

+875
-12
lines changed

‎.travis.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1-
sudo: false
21
language: node_js
32

43
node_js:
54
- "5.1"
65

7-
os:
8-
-osx
9-
-linux
6+
env:
7+
- CXX=g++-4.8
8+
9+
addons:
10+
apt:
11+
sources:
12+
- ubuntu-toolchain-r-test
13+
packages:
14+
- g++-4.8
1015

1116
install:
1217
- npm install

‎README.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,17 @@ First install:
8686
* Node.js (newer than 4.3.1)
8787
* Npm (newer 2.14.12)
8888

89+
In case you get a *node-gyp* error [follow the instrutions here](https://github.com/nodejs/node-gyp/blob/master/README.md) to fix it. The *vscode-textmate* package pulls in a native node dependency and those instructions will set up the node build tool which deals with those.
90+
8991
To **run and develop** do the following:
9092

9193
* Run `npm i`
9294
* Open in Visual Studio Code (`code .`)
9395
* *Optional:* run `tsc -w`, make code changes (on Windows, try `start node ".\node_modules\typescript\bin\tsc -w"`)
9496
* Press <kbd>F5</kbd> to debug
9597

98+
To **test** do the following: `npm run test`
99+
96100
### License
97-
The Microsoft C# extension is subject to [these license terms](RuntimeLicenses/license.txt).
98-
The source code to this extension is available on [https://github.com/OmniSharp/omnisharp-vscode](https://github.com/OmniSharp/omnisharp-vscode) and licensed under the [MIT license](LICENSE.txt).
101+
The Microsoft C# extension is subject to [these license terms](RuntimeLicenses/license.txt).
102+
The source code to this extension is available on [https://github.com/OmniSharp/omnisharp-vscode](https://github.com/OmniSharp/omnisharp-vscode) and licensed under the [MIT license](LICENSE.txt).

‎gulpfile.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ function installOmnisharp(omnisharps) {
4242

4343
return download.go(omni.flavor, omni.platform, log);
4444
});
45-
45+
4646
return Promise.all(promises);
4747
}
4848

@@ -53,7 +53,7 @@ function cleanOmnisharp() {
5353
gulp.task('omnisharp:clean', () => {
5454
return cleanOmnisharp();
5555
});
56-
56+
5757
gulp.task('omnisharp:install', ['omnisharp:clean'], () => {
5858
const flavor = gulpUtil.env.flavor || Flavor.CoreCLR;
5959
const platform = gulpUtil.env.platform || platform.getCurrentPlatform();
@@ -157,7 +157,7 @@ gulp.task('package:offline', ['clean'], () => {
157157

158158
/// Test Task
159159
gulp.task('test', () => {
160-
gulp.src('out/test/*.tests.js')
160+
gulp.src('out/test/**/*.tests.js')
161161
.pipe(mocha({ui: "tdd"}))
162162
.once('error', () => {
163163
process.exit(1);

‎package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"scripts": {
2222
"compile": "node ./node_modules/vscode/bin/compile -p ./",
2323
"watch": "node ./node_modules/vscode/bin/compile -watch -p ./",
24-
"test": "mocha --timeout 15000 -u tdd ./out/test/*.tests.js",
24+
"test": "mocha --timeout 15000 -u tdd ./out/test/**/*.tests.js",
2525
"postinstall": "node ./node_modules/vscode/bin/install"
2626
},
2727
"dependencies": {
@@ -46,7 +46,9 @@
4646
"tslint-microsoft-contrib": "^2.0.0",
4747
"typescript": "^1.7.3",
4848
"vscode": "^0.11.13",
49-
"vsce": "^1.7.0"
49+
"vsce": "^1.7.0",
50+
"chai": "^3.5.0",
51+
"vscode-textmate": "^2.1.1"
5052
},
5153
"engines": {
5254
"vscode": "^1.3.0"

‎src/features/codeLensProvider.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ export default class OmniSharpCodeLensProvider extends AbstractSupport implement
3232
};
3333

3434
provideCodeLenses(document: TextDocument, token: CancellationToken): CodeLens[] | Thenable<CodeLens[]> {
35-
let request = { Filename: document.fileName };
3635
return serverUtils.currentFileMembersAsTree(this._server, { Filename: document.fileName }, token).then(tree => {
3736
let ret: CodeLens[] = [];
3837
tree.TopLevelTypeDefinitions.forEach(node => this._convertQuickFix(ret, document.fileName, node));

‎test/syntaxes/class.tests.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import { should } from 'chai';
2+
import { Tokens, Token } from './utils/tokenizer';
3+
import { TokenizerUtil } from'./utils/tokenizerUtil';
4+
5+
describe("Grammar", function() {
6+
before(function() {
7+
should();
8+
});
9+
10+
describe("Class", function() {
11+
it("has a class keyword, a name and optional storage modifiers", function() {
12+
13+
const input = `
14+
namespace TestNamespace
15+
{
16+
public class PublicClass { }
17+
18+
class DefaultClass { }
19+
20+
internal class InternalClass { }
21+
22+
static class DefaultStaticClass { }
23+
24+
public static class PublicStaticClass { }
25+
26+
sealed class DefaultSealedClass { }
27+
28+
public sealed class PublicSealedClass { }
29+
30+
public abstract class PublicAbstractClass { }
31+
32+
abstract class DefaultAbstractClass { }
33+
}`;
34+
let tokens: Token[] = TokenizerUtil.tokenize(input);
35+
36+
tokens.should.contain(Tokens.StorageModifierKeyword("public", 4, 5));
37+
tokens.should.contain(Tokens.ClassKeyword("class", 4, 24));
38+
tokens.should.contain(Tokens.ClassIdentifier("PublicClass", 4, 30));
39+
40+
tokens.should.contain(Tokens.ClassKeyword("class", 6, 24));
41+
tokens.should.contain(Tokens.ClassIdentifier("DefaultClass", 6, 30));
42+
43+
tokens.should.contain(Tokens.StorageModifierKeyword("internal", 8, 5));
44+
tokens.should.contain(Tokens.ClassKeyword("class", 8, 24));
45+
tokens.should.contain(Tokens.ClassIdentifier("InternalClass", 8, 30));
46+
47+
tokens.should.contain(Tokens.StorageModifierKeyword("static", 10, 15));
48+
tokens.should.contain(Tokens.ClassKeyword("class", 10, 24));
49+
tokens.should.contain(Tokens.ClassIdentifier("DefaultStaticClass", 10, 30));
50+
51+
tokens.should.contain(Tokens.StorageModifierKeyword("public", 12, 5));
52+
tokens.should.contain(Tokens.StorageModifierKeyword("static", 12, 15));
53+
tokens.should.contain(Tokens.ClassKeyword("class", 12, 24));
54+
tokens.should.contain(Tokens.ClassIdentifier("PublicStaticClass", 12, 30));
55+
56+
tokens.should.contain(Tokens.StorageModifierKeyword("sealed", 14, 15));
57+
tokens.should.contain(Tokens.ClassKeyword("class", 14, 24));
58+
tokens.should.contain(Tokens.ClassIdentifier("DefaultSealedClass", 14, 30));
59+
60+
tokens.should.contain(Tokens.StorageModifierKeyword("public", 16, 5));
61+
tokens.should.contain(Tokens.StorageModifierKeyword("sealed", 16, 15));
62+
tokens.should.contain(Tokens.ClassKeyword("class", 16, 24));
63+
tokens.should.contain(Tokens.ClassIdentifier("PublicSealedClass", 16, 30));
64+
65+
tokens.should.contain(Tokens.StorageModifierKeyword("public", 18, 5));
66+
tokens.should.contain(Tokens.StorageModifierKeyword("abstract", 18, 15));
67+
tokens.should.contain(Tokens.ClassKeyword("class", 18, 24));
68+
tokens.should.contain(Tokens.ClassIdentifier("PublicAbstractClass", 18, 30));
69+
70+
tokens.should.contain(Tokens.StorageModifierKeyword("abstract", 20, 15));
71+
tokens.should.contain(Tokens.ClassKeyword("class", 20, 24));
72+
tokens.should.contain(Tokens.ClassIdentifier("DefaultAbstractClass", 20, 30));
73+
74+
});
75+
76+
});
77+
});
78+
79+

‎test/syntaxes/namespace.tests.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import { should } from 'chai';
2+
import { Tokens, Token } from './utils/tokenizer';
3+
import { TokenizerUtil } from'./utils/tokenizerUtil';
4+
5+
describe("Grammar", function() {
6+
before(function () {
7+
should();
8+
});
9+
10+
describe("Namespace", function() {
11+
it("has a namespace keyword and a name", function() {
12+
13+
const input = `
14+
namespace TestNamespace
15+
{
16+
}`;
17+
let tokens: Token[] = TokenizerUtil.tokenize(input);
18+
19+
tokens.should.contain(Tokens.NamespaceKeyword("namespace", 2, 1));
20+
tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 2, 11));
21+
});
22+
23+
it("can be nested", function() {
24+
25+
const input = `
26+
namespace TestNamespace
27+
{
28+
namespace NestedNamespace {
29+
30+
}
31+
}`;
32+
let tokens: Token[] = TokenizerUtil.tokenize(input);
33+
34+
tokens.should.contain(Tokens.NamespaceKeyword("namespace", 2, 1));
35+
tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 2, 11));
36+
37+
tokens.should.contain(Tokens.NamespaceKeyword("namespace", 4, 5));
38+
tokens.should.contain(Tokens.NamespaceIdentifier("NestedNamespace", 4, 15));
39+
});
40+
41+
it("can contain using statements", function() {
42+
43+
const input = `
44+
using UsineOne;
45+
using one = UsineOne.Something;
46+
47+
namespace TestNamespace
48+
{
49+
using UsingTwo;
50+
using two = UsineOne.Something;
51+
52+
namespace NestedNamespace
53+
{
54+
using UsingThree;
55+
using three = UsineOne.Something;
56+
}
57+
}`;
58+
let tokens: Token[] = TokenizerUtil.tokenize(input);
59+
60+
tokens.should.contain(Tokens.UsingKeyword("using", 2, 1));
61+
tokens.should.contain(Tokens.UsingKeyword("using", 3, 1));
62+
63+
tokens.should.contain(Tokens.NamespaceKeyword("namespace", 5, 1));
64+
tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 5, 11));
65+
66+
tokens.should.contain(Tokens.UsingKeyword("using", 7, 5));
67+
tokens.should.contain(Tokens.UsingKeyword("using", 8, 5));
68+
69+
tokens.should.contain(Tokens.NamespaceKeyword("namespace", 10, 5));
70+
tokens.should.contain(Tokens.NamespaceIdentifier("NestedNamespace", 10, 15));
71+
72+
tokens.should.contain(Tokens.UsingKeyword("using", 12, 9));
73+
tokens.should.contain(Tokens.UsingKeyword("using", 12, 9));
74+
});
75+
});
76+
});
77+
78+

‎test/syntaxes/utils/tokenizer.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import {ITokenizeLineResult, Registry, IGrammar, StackElement} from 'vscode-textmate';
2+
3+
export class Tokenizer
4+
{
5+
private _grammar : IGrammar;
6+
7+
constructor(grammarFilePath: string) {
8+
this._grammar = new Registry().loadGrammarFromPathSync(grammarFilePath);
9+
}
10+
11+
public tokenize(input: string): Token[] {
12+
let tokens: Token[] = [];
13+
14+
// ensure consistent line-endings irrelevant of OS
15+
input = input.replace("\r\n","\n");
16+
17+
let previousStack : StackElement = null;
18+
19+
const lines: string[] = input.split("\n");
20+
21+
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
22+
const line = lines[lineIndex];
23+
24+
let result: ITokenizeLineResult = this._grammar.tokenizeLine(line, previousStack);
25+
previousStack = result.ruleStack;
26+
27+
for (const token of result.tokens) {
28+
const text = line.substring(token.startIndex, token.endIndex);
29+
const type : string = token.scopes[token.scopes.length - 1];
30+
tokens.push(new Token(text, type, lineIndex+1, token.startIndex + 1));
31+
}
32+
}
33+
34+
return tokens;
35+
}
36+
}
37+
38+
export class Token {
39+
constructor(text: string, type: string, line?: number, column?: number) {
40+
this.text = text;
41+
this.type = type;
42+
this.column = column;
43+
this.line = line;
44+
}
45+
46+
public text: string;
47+
public type: string;
48+
public line: number;
49+
public column: number;
50+
}
51+
52+
export namespace Tokens {
53+
54+
function createToken(text: string, type: string, line?: number, column?: number) : Token {
55+
return new Token(text, type, line, column);
56+
}
57+
58+
export const NamespaceKeyword = (text: string, line?: number, column?: number) =>
59+
createToken(text, "keyword.other.namespace.cs", line, column);
60+
61+
export const NamespaceIdentifier = (text: string, line?: number, column?: number) =>
62+
createToken(text, "entity.name.type.namespace.cs", line, column);
63+
64+
export const UsingKeyword = (text: string, line?: number, column?: number) =>
65+
createToken(text, "keyword.other.using.cs", line, column);
66+
67+
export const ClassKeyword = (text: string, line?: number, column?: number) =>
68+
createToken(text, "storage.modifier.cs", line, column);
69+
70+
export const ClassIdentifier = (text: string, line?: number, column?: number) =>
71+
createToken(text, "entity.name.type.class.cs", line, column);
72+
73+
export const StorageModifierKeyword = (text: string, line?: number, column?: number) =>
74+
createToken(text, "storage.modifier.cs", line, column);
75+
76+
}
77+

‎test/syntaxes/utils/tokenizerUtil.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { Tokenizer, Token } from './tokenizer';
2+
3+
export class TokenizerUtil
4+
{
5+
private static _tokenizer: Tokenizer = new Tokenizer("syntaxes/csharp.json");
6+
7+
public static tokenize(input: string): Token[] {
8+
return TokenizerUtil._tokenizer.tokenize(input);
9+
}
10+
}

0 commit comments

Comments
 (0)