Merge pull request #482 from Coding/file-size-limit-tokenization

skipped tokenizer for lines longer than 10k characters

Author: Aaaaash

app/components/MonacoEditor/MonacoReact/grammars/set-grammars.js

@@ -1,3 +1,5 @@
+import config from 'config'
+import { notify, NOTIFY_TYPE } from 'components/Notification/actions'
 import languagesConfig from './languages'
 import { INITIAL } from './monaco-textmate'
 
@@ -43,15 +45,39 @@ export function wireTmGrammars (monaco, registry, languages) {
           monaco.languages.setTokensProvider(languageId, {
             getInitialState: () => new TokenizerState(INITIAL),
             tokenize: (line, state) => {
+              if (line.length > 10000) {
+                if (!config.tokenizationWarningAlreadyShown) {
+                  console.warn('Too many characters! Tokenization is skipped for lines longer than 10k characters for performance reasons.')
+                  notify({ message: i18n`editor.tokenizationWarning`, notifyType: NOTIFY_TYPE.ERROR })
+                  config.tokenizationWarningAlreadyShown = true
+                }
+                const tokens = new Uint32Array(2)
+                tokens[0] = 0
+                tokens[1] = (
+                  (1 << 0)
+                  | (0 << 8)
+                  | (0 << 11)
+                  | (1 << 14)
+                  | (2 << 23)
+                ) >>> 0;
+                return {
+                  endState: new TokenizerState(state.ruleStack),
+                  tokens: [{
+                    startIndex: 0,
+                    scopes: '',
+                    endIndex: line.length,
+                  }],
+                }
+              }
               const res = grammar.tokenizeLine(line, state.ruleStack)
-              return {
+              const tokenize = {
                 endState: new TokenizerState(res.ruleStack),
                 tokens: res.tokens.map(token => ({
                   ...token,
-                  // TODO: At the moment, monaco-editor doesn't seem to accept array of scopes
                   scopes: token.scopes[token.scopes.length - 1]
                 }))
               }
+              return tokenize
             }
           })
         } catch (e) {

app/config.js

@@ -41,6 +41,7 @@ const config = observable({
   // 是否为插件开发工作空间
   __PLUGIN_DEV__: false,
   fileicons: 'default',
+  tokenizationWarningAlreadyShown: false,
   tencentOrigin: 'https://studio.dev.tencent.com',
   get previewURL () {
     if (config.staticServingToken && config.spaceKey && config.staticServingURL) {

app/i18n/en_US/editor.json

@@ -0,0 +1,3 @@
+{
+  "tokenizationWarning": "Tokenization is skipped for lines longer than 10k characters for performance reasons."
+}

app/i18n/en_US/index.js

@@ -1,4 +1,4 @@
-const contents = ['menuBarItems', 'settings', 'file', 'panel', 'tab', 'git', 'fileTree', 'global', 'modal', 'login', 'import', 'fileTreeTool', 'commandPalette', 'project', 'monaco', 'plugin'];
+const contents = ['menuBarItems', 'settings', 'file', 'panel', 'tab', 'git', 'fileTree', 'global', 'modal', 'login', 'import', 'fileTreeTool', 'commandPalette', 'project', 'monaco', 'plugin', 'editor'];
 
 export default contents.reduce((p, v) => {
   p[v] = require(`./${v}.json`)

app/i18n/zh_CN/editor.json

@@ -0,0 +1,3 @@
+{
+  "tokenizationWarning": "由于性能原因，将不对单行大于1万字符的文件进行标记化解析."
+}

app/i18n/zh_CN/index.js

@@ -1,4 +1,4 @@
-const contents = ['menuBarItems', 'settings', 'file', 'panel', 'tab', 'git', 'fileTree', 'global', 'modal', 'login', 'import', 'fileTreeTool', 'commandPalette', 'project', 'monaco', 'plugin'];
+const contents = ['menuBarItems', 'settings', 'file', 'panel', 'tab', 'git', 'fileTree', 'global', 'modal', 'login', 'import', 'fileTreeTool', 'commandPalette', 'project', 'monaco', 'plugin', 'editor'];
 
 export default contents.reduce((p, v) => {
   p[v] = require(`./${v}.json`)