(perf) cap full file semantic tokens (#774)

#772
Cap the full file semantic token request to 50000. If the text length of the svelte2tsx generated code is bigger than that, return null and let the client request for ranged semantic tokens, that is what's visible on the screen.

Author: jasonlyu123

packages/language-server/src/plugins/PluginHost.ts

@@ -408,14 +408,10 @@ export class PluginHost implements LSProvider, OnWatchFileChanges {
             throw new Error('Cannot call methods on an unopened document');
         }
 
-        return (
-            (await this.execute<SemanticTokens>(
-                'getSemanticTokens',
-                [document, range],
-                ExecuteMode.FirstNonNull
-            )) ?? {
-                data: []
-            }
+        return await this.execute<SemanticTokens>(
+            'getSemanticTokens',
+            [document, range],
+            ExecuteMode.FirstNonNull
         );
     }
 

packages/language-server/src/plugins/interfaces.ts

@@ -148,7 +148,7 @@ export interface SelectionRangeProvider {
 }
 
 export interface SemanticTokensProvider {
-    getSemanticTokens(textDocument: Document, range?: Range): Resolvable<SemanticTokens>;
+    getSemanticTokens(textDocument: Document, range?: Range): Resolvable<SemanticTokens | null>;
 }
 
 export interface OnWatchFileChangesPara {

packages/language-server/src/plugins/typescript/TypeScriptPlugin.ts

@@ -413,7 +413,7 @@ export class TypeScriptPlugin
         return this.signatureHelpProvider.getSignatureHelp(document, position, context);
     }
 
-    async getSemanticTokens(textDocument: Document, range?: Range): Promise<SemanticTokens> {
+    async getSemanticTokens(textDocument: Document, range?: Range): Promise<SemanticTokens | null> {
         if (!this.featureEnabled('semanticTokens')) {
             return {
                 data: []

packages/language-server/src/plugins/typescript/features/SemanticTokensProvider.ts

@@ -6,12 +6,20 @@ import { SnapshotFragment } from '../DocumentSnapshot';
 import { LSAndTSDocResolver } from '../LSAndTSDocResolver';
 import { convertToTextSpan } from '../utils';
 
+const CONTENT_LENGTH_LIMIT = 50000;
+
 export class SemanticTokensProviderImpl implements SemanticTokensProvider {
     constructor(private readonly lsAndTsDocResolver: LSAndTSDocResolver) {}
 
-    async getSemanticTokens(textDocument: Document, range?: Range): Promise<SemanticTokens> {
+    async getSemanticTokens(textDocument: Document, range?: Range): Promise<SemanticTokens | null> {
         const { lang, tsDoc } = this.lsAndTsDocResolver.getLSAndTSDoc(textDocument);
         const fragment = await tsDoc.getFragment();
+
+        // for better performance, don't do full-file semantic tokens when the file is too big
+        if (!range && fragment.text.length > CONTENT_LENGTH_LIMIT) {
+            return null;
+        }
+
         const textSpan = range
             ? convertToTextSpan(range, fragment)
             : {

packages/language-server/src/server.ts

@@ -12,7 +12,9 @@ import {
     TextDocumentIdentifier,
     TextDocumentPositionParams,
     TextDocumentSyncKind,
-    WorkspaceEdit
+    WorkspaceEdit,
+    SemanticTokensRequest,
+    SemanticTokensRangeRequest
 } from 'vscode-languageserver';
 import { IPCMessageReader, IPCMessageWriter, createConnection } from 'vscode-languageserver/node';
 import { DiagnosticsManager } from './lib/DiagnosticsManager';
@@ -304,8 +306,10 @@ export function startServer(options?: LSOptions) {
         updateAllDiagnostics();
     });
 
-    connection.languages.semanticTokens.on((evt) => pluginHost.getSemanticTokens(evt.textDocument));
-    connection.languages.semanticTokens.onRange((evt) =>
+    connection.onRequest(SemanticTokensRequest.type, (evt) =>
+        pluginHost.getSemanticTokens(evt.textDocument)
+    );
+    connection.onRequest(SemanticTokensRangeRequest.type, (evt) =>
         pluginHost.getSemanticTokens(evt.textDocument, evt.range)
     );
 

packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts

@@ -33,18 +33,22 @@ describe('SemanticTokensProvider', () => {
     it('provides semantic token', async () => {
         const { provider, document } = setup();
 
-        const { data } = await provider.getSemanticTokens(document);
+        const { data } = (await provider.getSemanticTokens(document)) ?? {
+            data: []
+        };
 
         assertResult(data, getExpected(/* isFull */ true));
     });
 
     it('provides partial semantic token', async () => {
         const { provider, document } = setup();
 
-        const { data } = await provider.getSemanticTokens(
+        const { data } = (await provider.getSemanticTokens(
             document,
             Range.create(Position.create(0, 0), Position.create(9, 0))
-        );
+        )) ?? {
+            data: []
+        };
 
         assertResult(data, getExpected(/* isFull */ false));
     });