Deprecate 'semantic_tokens' config in favor of 'augmentsSyntaxTokens'

The `augmentsSyntaxTokens` client capability can be used to indicate
that the client is going to combine existing syntax tokens with the
semantic tokens of the server. This can not only reduce the amount of
data that is transferred over the protocol but can also lead to better
highlighting as syntax tokens often more accurately distinguish between
different language constructs on the syntax level. An example would be
keywords where the LSP protocol has a single standard token type
compared to syntax highlighting where different keywords are often
differentiated between based on their purpose.

The 'semantic_tokens' config had been added for this purpose. The
'partial' option is used to prevent LSP semantic tokens to override
existing syntax highlighting of the editor. The benefit of the client
capability is that this decision is automatically made by supported
editors instead of relying on user to select the right configuration.

Author: Techatrix

schema.json

@@ -33,7 +33,7 @@
             "default": []
         },
         "semantic_tokens": {
-            "description": "Set level of semantic tokens. `partial` only includes information that requires semantic analysis.",
+            "description": "Deprecated. The client should set the 'augmentsSyntaxTokens' capability.\n\nSet level of semantic tokens. `partial` only includes information that requires semantic analysis.",
             "type": "string",
             "enum": [
                 "none",

src/Config.zig

@@ -23,6 +23,8 @@ enable_build_on_save: ?bool = null,
 /// If the `build.zig` has declared a 'check' step, it will be preferred over the default 'install' step.
 build_on_save_args: []const []const u8 = &.{},
 
+/// Deprecated. The client should set the 'augmentsSyntaxTokens' capability.
+///
 /// Set level of semantic tokens. `partial` only includes information that requires semantic analysis.
 semantic_tokens: enum {
     none,

src/Server.zig

@@ -72,6 +72,7 @@ const ClientCapabilities = struct {
     supports_publish_diagnostics: bool = false,
     supports_code_action_fixall: bool = false,
     supports_semantic_tokens_overlapping: bool = false,
+    semantic_tokens_augment_syntax_tokens: bool = false,
     hover_supports_md: bool = false,
     signature_help_supports_md: bool = false,
     completion_doc_supports_md: bool = false,
@@ -458,6 +459,7 @@ fn initializeHandler(server: *Server, arena: std.mem.Allocator, request: types.I
         }
         if (textDocument.semanticTokens) |semanticTokens| {
             server.client_capabilities.supports_semantic_tokens_overlapping = semanticTokens.overlappingTokenSupport orelse false;
+            server.client_capabilities.semantic_tokens_augment_syntax_tokens = semanticTokens.augmentsSyntaxTokens orelse false;
         }
     }
 
@@ -1254,7 +1256,7 @@ fn semanticTokensFullHandler(server: *Server, arena: std.mem.Allocator, request:
         handle,
         null,
         server.offset_encoding,
-        server.config_manager.config.semantic_tokens == .partial,
+        server.client_capabilities.semantic_tokens_augment_syntax_tokens or server.config_manager.config.semantic_tokens == .partial,
         server.client_capabilities.supports_semantic_tokens_overlapping,
     );
 }
@@ -1281,7 +1283,7 @@ fn semanticTokensRangeHandler(server: *Server, arena: std.mem.Allocator, request
         handle,
         loc,
         server.offset_encoding,
-        server.config_manager.config.semantic_tokens == .partial,
+        server.client_capabilities.semantic_tokens_augment_syntax_tokens or server.config_manager.config.semantic_tokens == .partial,
         server.client_capabilities.supports_semantic_tokens_overlapping,
     );
 }

src/tools/config.json

@@ -32,7 +32,7 @@
         },
         {
             "name": "semantic_tokens",
-            "description": "Set level of semantic tokens. `partial` only includes information that requires semantic analysis.",
+            "description": "Deprecated. The client should set the 'augmentsSyntaxTokens' capability.\n\nSet level of semantic tokens. `partial` only includes information that requires semantic analysis.",
             "type": "enum",
             "enum": [
                 "none",