Add feature flag to turn on the new Roslyn tokenizer (dotnet#11185)

This makes the feature work in VS and VS Code (with corresponding VS
Code change to pass the right flag, coming once this is merged)

VS Code PR: dotnet/vscode-csharp#7748

/cc @danroth27, @leslierichardson95, @333fred, @chsienki and anyone else
who wants, for wording:
VS Code:

https://github.com/dotnet/vscode-csharp/pull/7748/files#diff-9b29d8b47a74097a04da353b72a8632d485d658282457b97b4caf26575a42dd0R131
VS:

https://github.com/dotnet/razor/pull/11185/files#diff-18b9b146f4d27a6a5b6248ece31b30570d84c2a409b25bf5b660de86e608fa4cR80

Author: davidwengier

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/DefaultLanguageServerFeatureOptions.cs

@@ -38,4 +38,6 @@ internal class DefaultLanguageServerFeatureOptions : LanguageServerFeatureOption
     public override bool DisableRazorLanguageServer => false;
 
     public override bool ForceRuntimeCodeGeneration => false;
+
+    public override bool UseRoslynTokenizer => false;
 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Hosting/ConfigurableLanguageServerFeatureOptions.cs

@@ -23,6 +23,7 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     private readonly bool? _useRazorCohostServer;
     private readonly bool? _disableRazorLanguageServer;
     private readonly bool? _forceRuntimeCodeGeneration;
+    private readonly bool? _useRoslynTokenizer;
 
     public override bool SupportsFileManipulation => _supportsFileManipulation ?? _defaults.SupportsFileManipulation;
     public override string CSharpVirtualDocumentSuffix => _csharpVirtualDocumentSuffix ?? DefaultLanguageServerFeatureOptions.DefaultCSharpVirtualDocumentSuffix;
@@ -37,6 +38,7 @@ internal class ConfigurableLanguageServerFeatureOptions : LanguageServerFeatureO
     public override bool UseRazorCohostServer => _useRazorCohostServer ?? _defaults.UseRazorCohostServer;
     public override bool DisableRazorLanguageServer => _disableRazorLanguageServer ?? _defaults.DisableRazorLanguageServer;
     public override bool ForceRuntimeCodeGeneration => _forceRuntimeCodeGeneration ?? _defaults.ForceRuntimeCodeGeneration;
+    public override bool UseRoslynTokenizer => _useRoslynTokenizer ?? _defaults.UseRoslynTokenizer;
 
     public ConfigurableLanguageServerFeatureOptions(string[] args)
     {
@@ -60,6 +62,7 @@ public ConfigurableLanguageServerFeatureOptions(string[] args)
             TryProcessBoolOption(nameof(UseRazorCohostServer), ref _useRazorCohostServer, option, args, i);
             TryProcessBoolOption(nameof(DisableRazorLanguageServer), ref _disableRazorLanguageServer, option, args, i);
             TryProcessBoolOption(nameof(ForceRuntimeCodeGeneration), ref _forceRuntimeCodeGeneration, option, args, i);
+            TryProcessBoolOption(nameof(UseRoslynTokenizer), ref _useRoslynTokenizer, option, args, i);
         }
     }
 

src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/LanguageServerFeatureOptions.cs

@@ -36,7 +36,9 @@ internal abstract class LanguageServerFeatureOptions
     public abstract bool DisableRazorLanguageServer { get; }
 
     /// <summary>
-    /// When enabled, design time code will not be generated. All tooling will be using runtime code generation.
+    /// When enabled, design time code will not be generated. All tooling, except formatting, will be using runtime code generation.
     /// </summary>
     public abstract bool ForceRuntimeCodeGeneration { get; }
+
+    public abstract bool UseRoslynTokenizer { get; }
 }

src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/ProjectSystem/ProjectState.cs

@@ -14,6 +14,7 @@
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Razor.Workspaces;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 
 namespace Microsoft.CodeAnalysis.Razor.ProjectSystem;
 
@@ -171,12 +172,14 @@ RazorProjectEngine CreateProjectEngine()
             {
                 var configuration = HostProject.Configuration;
                 var rootDirectoryPath = Path.GetDirectoryName(HostProject.FilePath).AssumeNotNull();
+                var useRoslynTokenizer = LanguageServerFeatureOptions.UseRoslynTokenizer;
 
                 return _projectEngineFactoryProvider.Create(configuration, rootDirectoryPath, builder =>
                 {
                     builder.SetRootNamespace(HostProject.RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, CSharpParseOptions.Default));
                 });
             }
         }

src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Remote/RemoteClientInitializationOptions.cs

@@ -33,4 +33,7 @@ internal struct RemoteClientInitializationOptions
 
     [JsonPropertyName("showAllCSharpCodeActions")]
     public required bool ShowAllCSharpCodeActions { get; set; }
+
+    [JsonPropertyName("useRoslynTokenizer")]
+    public required bool UseRoslynTokenizer { get; set; }
 }

src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/Initialization/RemoteLanguageServerFeatureOptions.cs

@@ -42,4 +42,6 @@ internal class RemoteLanguageServerFeatureOptions : LanguageServerFeatureOptions
     public override bool DisableRazorLanguageServer => throw new InvalidOperationException("This option has not been synced to OOP.");
 
     public override bool ForceRuntimeCodeGeneration => _options.ForceRuntimeCodeGeneration;
+
+    public override bool UseRoslynTokenizer => _options.UseRoslynTokenizer;
 }

src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/ProjectSystem/RemoteProjectSnapshot.cs

@@ -19,6 +19,7 @@
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 using Microsoft.VisualStudio.Threading;
 
 namespace Microsoft.CodeAnalysis.Remote.Razor.ProjectSystem;
@@ -51,6 +52,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
         _lazyProjectEngine = new AsyncLazy<RazorProjectEngine>(async () =>
         {
             var configuration = await _lazyConfiguration.GetValueAsync();
+            var useRoslynTokenizer = SolutionSnapshot.SnapshotManager.LanguageServerFeatureOptions.UseRoslynTokenizer;
             return ProjectEngineFactories.DefaultProvider.Create(
                 configuration,
                 rootDirectoryPath: Path.GetDirectoryName(FilePath).AssumeNotNull(),
@@ -59,6 +61,7 @@ public RemoteProjectSnapshot(Project project, RemoteSolutionSnapshot solutionSna
                     builder.SetRootNamespace(RootNamespace);
                     builder.SetCSharpLanguageVersion(CSharpLanguageVersion);
                     builder.SetSupportLocalizedComponentNames();
+                    builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, CSharpParseOptions.Default));
                 });
         },
         joinableTaskFactory: null);

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/Remote/RemoteServiceInvoker.cs

@@ -141,6 +141,7 @@ private async Task InitializeRemoteClientAsync(RazorRemoteHostClient remoteClien
                     ForceRuntimeCodeGeneration = _languageServerFeatureOptions.ForceRuntimeCodeGeneration,
                     SupportsFileManipulation = _languageServerFeatureOptions.SupportsFileManipulation,
                     ShowAllCSharpCodeActions = _languageServerFeatureOptions.ShowAllCSharpCodeActions,
+                    UseRoslynTokenizer = _languageServerFeatureOptions.UseRoslynTokenizer,
                 };
 
                 _logger.LogDebug($"First OOP call, so initializing OOP service.");

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/VisualStudioLanguageServerFeatureOptions.cs

@@ -19,6 +19,7 @@ internal class VisualStudioLanguageServerFeatureOptions : LanguageServerFeatureO
     private readonly Lazy<bool> _useRazorCohostServer;
     private readonly Lazy<bool> _disableRazorLanguageServer;
     private readonly Lazy<bool> _forceRuntimeCodeGeneration;
+    private readonly Lazy<bool> _useRoslynTokenizer;
 
     [ImportingConstructor]
     public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEditorFeatureDetector)
@@ -71,12 +72,18 @@ public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEdi
             var forceRuntimeCodeGeneration = featureFlags.IsFeatureEnabled(WellKnownFeatureFlagNames.ForceRuntimeCodeGeneration, defaultValue: false);
             return forceRuntimeCodeGeneration;
         });
+
+        _useRoslynTokenizer = new Lazy<bool>(() =>
+        {
+            var featureFlags = (IVsFeatureFlags)Package.GetGlobalService(typeof(SVsFeatureFlags));
+            var useRoslynTokenizer = featureFlags.IsFeatureEnabled(WellKnownFeatureFlagNames.UseRoslynTokenizer, defaultValue: false);
+            return useRoslynTokenizer;
+        });
     }
 
     // We don't currently support file creation operations on VS Codespaces or VS Liveshare
     public override bool SupportsFileManipulation => !IsCodespacesOrLiveshare;
 
-
     public override string CSharpVirtualDocumentSuffix => ".ide.g.cs";
 
     public override string HtmlVirtualDocumentSuffix => "__virtual.html";
@@ -103,4 +110,6 @@ public VisualStudioLanguageServerFeatureOptions(ILspEditorFeatureDetector lspEdi
 
     /// <inheritdoc />
     public override bool ForceRuntimeCodeGeneration => _forceRuntimeCodeGeneration.Value;
+
+    public override bool UseRoslynTokenizer => _useRoslynTokenizer.Value;
 }

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/WellKnownFeatureFlagNames.cs

@@ -11,4 +11,5 @@ internal static class WellKnownFeatureFlagNames
     public const string UseRazorCohostServer = "Razor.LSP.UseRazorCohostServer";
     public const string DisableRazorLanguageServer = "Razor.LSP.DisableRazorLanguageServer";
     public const string ForceRuntimeCodeGeneration = "Razor.LSP.ForceRuntimeCodeGeneration";
+    public const string UseRoslynTokenizer = "Razor.LSP.UseRoslynTokenizer";
 }