Add telemtry for how long it takes to parse files with tree-sitter (microsoft#213565)

* Make space for tree sitter

* Add the tree sitter wasm file

* Very naive tree-sitter syntax highlighting for html, with a layer breaker

* Update tree when content changes

* WIP for making abstract tokens class

* Handle theme changes

* Replace entire text model value with parse callback

* Perf improvements

* Add tree-sitter-typescript

* Add typescript + better initial parsing

* Refactor into tree parsing service and fix flaw in parse callback

* Remove things that aren't the parser service

* Add yielding

* Remove changes that aren't required for PR

* Remove more file changes

* Reduce yield to 50 ms

* Fix incremental parsing

* Try update node-abi

* Revert "Try update node-abi"

This reverts commit df28801.

* Update text buffer chunk api

* fix build

* Remove tree-sitter dependency

* Adopt new, as yet unpublished, `@vscode/tree-sitter-wasm` package

* Use published `@vscode/tree-sitter-wasm` package

* Break `TreeSitterTree` and `TreeSitterParserService` into better pieces
and:
- document the order of editor changes
- use service injection where `TextModel` is constructed

* Fix tests

* Remove unneeded import

* Fix missing tree-sitter-wasm in web and remote

* Make package.jsons match

* Add @vscode/tree-sitter-wasm to web loader config

* Try using importAMDNodeModule

* PR feedback

* Add race condition test for changing language while loading language

* Use same timeout

* Queue content changes

* Remove override dispose

* Move queue into TreeSitterTree

---------

Co-authored-by: Peng Lyu <[email protected]>

Author: alexr00

.eslintrc.json

@@ -755,7 +755,8 @@
 							"vs/base/~",
 							"vs/base/parts/*/~",
 							"vs/platform/*/~",
-							"vs/editor/~"
+							"vs/editor/~",
+							"@vscode/tree-sitter-wasm" // node module allowed even in /common/
 						]
 					},
 					{

build/.webignore

@@ -38,6 +38,7 @@ vscode-textmate/webpack.config.js
 
 # This makes sure the model is included in the package
 !@vscode/vscode-languagedetection/model/**
+!@vscode/tree-sitter-wasm/wasm/**
 
 # Ensure only the required telemetry pieces are loaded in web to reduce bundle size
 @microsoft/1ds-core-js/**

package.json

@@ -78,6 +78,7 @@
     "@vscode/spdlog": "^0.15.0",
     "@vscode/sqlite3": "5.1.6-vscode",
     "@vscode/sudo-prompt": "9.3.1",
+    "@vscode/tree-sitter-wasm": "^0.0.1",
     "@vscode/vscode-languagedetection": "1.0.21",
     "@vscode/windows-mutex": "^0.5.0",
     "@vscode/windows-process-tree": "^0.6.0",

remote/package.json

@@ -11,6 +11,7 @@
     "@vscode/proxy-agent": "^0.22.0",
     "@vscode/ripgrep": "^1.15.9",
     "@vscode/spdlog": "^0.15.0",
+    "@vscode/tree-sitter-wasm": "^0.0.1",
     "@vscode/vscode-languagedetection": "1.0.21",
     "@vscode/windows-process-tree": "^0.6.0",
     "@vscode/windows-registry": "^1.1.0",

remote/web/package.json

@@ -6,6 +6,7 @@
 		"@microsoft/1ds-core-js": "^3.2.13",
 		"@microsoft/1ds-post-js": "^3.2.13",
 		"@vscode/iconv-lite-umd": "0.7.0",
+		"@vscode/tree-sitter-wasm": "^0.0.1",
 		"@vscode/vscode-languagedetection": "1.0.21",
 		"@xterm/addon-clipboard": "0.2.0-beta.34",
 		"@xterm/addon-image": "0.9.0-beta.51",

remote/web/yarn.lock

@@ -43,6 +43,11 @@
   resolved "https://registry.yarnpkg.com/@vscode/iconv-lite-umd/-/iconv-lite-umd-0.7.0.tgz#d2f1e0664ee6036408f9743fee264ea0699b0e48"
   integrity sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg==
 
+"@vscode/tree-sitter-wasm@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.1.tgz#ffb2e295a416698f4c77cbffeca3b28567d6754b"
+  integrity sha512-m0GKnQ3BxWnVd+20KLGwr1+Qvt/RiiaJmKAqHNU35pNydDtduUzyBm7ETz/T0vOVKoeIAaiYsJOA1aKWs7Y1tA==
+
 "@vscode/[email protected]":
   version "1.0.21"
   resolved "https://registry.yarnpkg.com/@vscode/vscode-languagedetection/-/vscode-languagedetection-1.0.21.tgz#89b48f293f6aa3341bb888c1118d16ff13b032d3"

remote/yarn.lock

@@ -98,6 +98,11 @@
     mkdirp "^1.0.4"
     node-addon-api "7.1.0"
 
+"@vscode/tree-sitter-wasm@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.1.tgz#ffb2e295a416698f4c77cbffeca3b28567d6754b"
+  integrity sha512-m0GKnQ3BxWnVd+20KLGwr1+Qvt/RiiaJmKAqHNU35pNydDtduUzyBm7ETz/T0vOVKoeIAaiYsJOA1aKWs7Y1tA==
+
 "@vscode/[email protected]":
   version "1.0.21"
   resolved "https://registry.yarnpkg.com/@vscode/vscode-languagedetection/-/vscode-languagedetection-1.0.21.tgz#89b48f293f6aa3341bb888c1118d16ff13b032d3"

src/bootstrap-window.js

@@ -197,6 +197,7 @@ const isESM = false;
 			// using a fallback such as node.js require which does not exist in sandbox
 			const baseNodeModulesPath = isDev ? '../node_modules' : '../node_modules.asar';
 			loaderConfig.paths = {
+				'@vscode/tree-sitter-wasm': `${baseNodeModulesPath}/@vscode/tree-sitter-wasm/wasm/tree-sitter.js`,
 				'vscode-textmate': `${baseNodeModulesPath}/vscode-textmate/release/main.js`,
 				'vscode-oniguruma': `${baseNodeModulesPath}/vscode-oniguruma/release/main.js`,
 				'vsda': `${baseNodeModulesPath}/vsda/index.js`,