Pass UseRoslynTokenizer setting to the LSP server

Author: davidwengier

package.json

@@ -39,7 +39,7 @@
   "defaults": {
     "roslyn": "4.13.0-2.24531.3",
     "omniSharp": "1.39.11",
-    "razor": "9.0.0-preview.24531.4",
+    "razor": "9.0.0-preview.24555.12",
     "razorOmnisharp": "7.0.0-preview.23363.1",
     "xamlTools": "17.13.35506.24"
   },
@@ -1528,6 +1528,13 @@
             "description": "%configuration.razor.languageServer.forceRuntimeCodeGeneration%",
             "order": 90
           },
+          "razor.languageServer.useRoslynTokenizer": {
+            "type": "boolean",
+            "scope": "machine-overridable",
+            "default": false,
+            "markdownDescription": "%configuration.razor.languageServer.useRoslynTokenizer%",
+            "order": 90
+          },
           "razor.languageServer.suppressLspErrorToasts": {
             "type": "boolean",
             "default": true,

package.nls.json

@@ -127,6 +127,12 @@
   "configuration.razor.languageServer.debug": "Specifies whether to wait for debug attach when launching the language server.",
   "configuration.razor.server.trace": "Specifies the logging level to use for the Razor server.",
   "configuration.razor.languageServer.forceRuntimeCodeGeneration": "(EXPERIMENTAL) Enable combined design time/runtime code generation for Razor files",
+  "configuration.razor.languageServer.useRoslynTokenizer": {
+    "message": "(EXPERIMENTAL) Use the C# tokenizer for Razor files in the IDE. Enables some new C# features, like interpolated and raw strings, in Razor files opened in Visual Studio. This matches using `<features>use-roslyn-tokenizer</feature>` in a `.csproj` file for command line builds, and may result in inconsistencies if this option and your project files do not match.",
+    "comment": [
+      "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+    ]
+  },
   "configuration.razor.languageServer.suppressLspErrorToasts": "Suppresses error toasts from showing up if the server encounters a recoverable error.",
   "debuggers.coreclr.configurationSnippets.label.console-local": ".NET: Launch Executable file (Console)",
   "debuggers.coreclr.configurationSnippets.label.web-local": ".NET: Launch Executable file (Web)",

src/razor/src/razorLanguageServerClient.ts

@@ -271,6 +271,11 @@ export class RazorLanguageServerClient implements vscode.Disposable {
                 args.push('true');
             }
 
+            if (options.useRoslynTokenizer) {
+                args.push('--UseRoslynTokenizer');
+                args.push('true');
+            }
+
             if (this.telemetryExtensionDllPath.length > 0) {
                 args.push('--telemetryLevel', this.vscodeTelemetryReporter.telemetryLevel);
                 args.push('--sessionId', getSessionId());

src/razor/src/razorLanguageServerOptions.ts

@@ -13,5 +13,6 @@ export interface RazorLanguageServerOptions {
     logLevel: LogLevel;
     usingOmniSharp: boolean;
     forceRuntimeCodeGeneration: boolean;
+    useRoslynTokenizer: boolean;
     suppressErrorToasts: boolean;
 }

src/razor/src/razorLanguageServerOptionsResolver.ts

@@ -25,6 +25,7 @@ export function resolveRazorLanguageServerOptions(
     const usingOmniSharp =
         !getCSharpDevKit() && vscodeApi.workspace.getConfiguration().get<boolean>('dotnet.server.useOmnisharp');
     const forceRuntimeCodeGeneration = serverConfig.get<boolean>('forceRuntimeCodeGeneration');
+    const useRoslynTokenizer = serverConfig.get<boolean>('useRoslynTokenizer');
     const suppressErrorToasts = serverConfig.get<boolean>('suppressLspErrorToasts');
 
     return {
@@ -34,6 +35,7 @@ export function resolveRazorLanguageServerOptions(
         outputChannel: logger.outputChannel,
         usingOmniSharp,
         forceRuntimeCodeGeneration,
+        useRoslynTokenizer,
         suppressErrorToasts,
     } as RazorLanguageServerOptions;
 }