Fix how semantic tokens server capabilities are computed (#213)

alcarney · danixeee · web-flow · commit 5e8a8f0793cf · 2021-11-06T13:58:26.000+01:00
* Fix server capabilities for semantic tokens.

A server can support up to three separate semantic tokens requests.
Rather than have `@server.feature()` methods provide the full
`SemanticTokensOptions` definition, this commit adjusts the
`LSP_METHODS_MAP` to accept just the `SemanticTokensLengend`.

Then in the `ServerCapabilitiesBuilder`, the `full` and `range` fields
of `SemanticTokensOptions` are computed according to which features are
present.

Since `SemanticTokensOptions` only supports a single legend, if multiple
legends are found, only the first one will be used.

* Add semantic tokens example

This updates the example `json-extension` to include an example
`textDocument/semanticTokens/full` implementation that highlights all
keys in a JSON document.

* Update changelog

Co-authored-by: Daniel Elero &lt;danixeee@gmail.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,19 +11,16 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Changed
 
-### Fixed
-
-## [0.11.3] - 09/30/2021
-
-### Added
-
-### Changed
+- Update json-example to include an example semantic tokens method ([204])
 
 ### Fixed
 
 - Fix example extension client not detecting debug mode appropriately ([#193])
+- Fix how the `semantic_tokens_provider` field of `ServerCapabilities` is computed ([213])
 
 [#193]: https://github.com/openlawlibrary/pygls/issues/193
+[#204]: https://github.com/openlawlibrary/pygls/issues/204
+[#213]: https://github.com/openlawlibrary/pygls/pulls/213
 
 ## [0.11.2] - 07/23/2021
 
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -1,6 +1,7 @@
 # Contributors (alphabetical)
 
 - [@augb](https://github.com/augb)
+- [Alex Carney](https://github.com/alcarney)
 - [Brett Cannon](https://github.com/brettcannon/)
 - [Daniel Elero](https://github.com/danixeee)
 - [Daniel Miller](https://github.com/millerdev)
diff --git a/examples/json-extension/server/server.py b/examples/json-extension/server/server.py
@@ -16,20 +16,23 @@
 ############################################################################
 import asyncio
 import json
+import re
 import time
 import uuid
 from json import JSONDecodeError
 from typing import Optional
 
 from pygls.lsp.methods import (COMPLETION, TEXT_DOCUMENT_DID_CHANGE,
-                               TEXT_DOCUMENT_DID_CLOSE, TEXT_DOCUMENT_DID_OPEN)
+                               TEXT_DOCUMENT_DID_CLOSE, TEXT_DOCUMENT_DID_OPEN, 
+                               TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL)
 from pygls.lsp.types import (CompletionItem, CompletionList, CompletionOptions,
                              CompletionParams, ConfigurationItem,
                              ConfigurationParams, Diagnostic,
                              DidChangeTextDocumentParams,
                              DidCloseTextDocumentParams,
                              DidOpenTextDocumentParams, MessageType, Position,
                              Range, Registration, RegistrationParams,
+                             SemanticTokens, SemanticTokensLegend, SemanticTokensParams,
                              Unregistration, UnregistrationParams)
 from pygls.lsp.types.basic_structures import (WorkDoneProgressBegin,
                                               WorkDoneProgressEnd,
@@ -151,6 +154,47 @@ async def did_open(ls, params: DidOpenTextDocumentParams):
     _validate(ls, params)
 
 
+@json_server.feature(
+    TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+    SemanticTokensLegend(
+        token_types = ["operator"],
+        token_modifiers = []
+    )
+)
+def semantic_tokens(ls: JsonLanguageServer, params: SemanticTokensParams):
+    """See https://microsoft.github.io/language-server-protocol/specification#textDocument_semanticTokens
+    for details on how semantic tokens are encoded."""
+    
+    TOKENS = re.compile('".*"(?=:)')
+    
+    uri = params.text_document.uri
+    doc = ls.workspace.get_document(uri)
+
+    last_line = 0
+    last_start = 0
+
+    data = []
+
+    for lineno, line in enumerate(doc.lines):
+        last_start = 0
+
+        for match in TOKENS.finditer(line):
+            start, end = match.span()
+            data += [
+                (lineno - last_line),
+                (start - last_start),
+                (end - start),
+                0, 
+                0
+            ]
+
+            last_line = lineno
+            last_start = start
+
+    return SemanticTokens(data=data)
+
+
+
 @json_server.command(JsonLanguageServer.CMD_PROGRESS)
 async def progress(ls: JsonLanguageServer, *args):
     """Create and start the progress on the client."""
diff --git a/pygls/capabilities.py b/pygls/capabilities.py
@@ -22,20 +22,22 @@
                                TEXT_DOCUMENT_CALL_HIERARCHY_PREPARE, TEXT_DOCUMENT_DID_CLOSE,
                                TEXT_DOCUMENT_DID_OPEN, TEXT_DOCUMENT_DID_SAVE,
                                TEXT_DOCUMENT_LINKED_EDITING_RANGE, TEXT_DOCUMENT_MONIKER,
-                               TEXT_DOCUMENT_SEMANTIC_TOKENS, TEXT_DOCUMENT_WILL_SAVE,
+                               TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+                               TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL_DELTA,
+                               TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE, TEXT_DOCUMENT_WILL_SAVE,
                                TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, TYPE_DEFINITION,
                                WORKSPACE_DID_CREATE_FILES, WORKSPACE_DID_DELETE_FILES,
                                WORKSPACE_DID_RENAME_FILES, WORKSPACE_SYMBOL,
                                WORKSPACE_WILL_CREATE_FILES, WORKSPACE_WILL_DELETE_FILES,
                                WORKSPACE_WILL_RENAME_FILES)
 from pygls.lsp.types import (CodeLensOptions, CompletionOptions, DocumentLinkOptions,
                              ExecuteCommandOptions, ImplementationOptions, SaveOptions,
+                             SemanticTokensOptions, SemanticTokensRegistrationOptions,
+                             SemanticTokensRequestsFull,
                              ServerCapabilities, SignatureHelpOptions,
                              TextDocumentSyncOptionsServerCapabilities, TypeDefinitionOptions,
                              WorkspaceFileOperationsServerCapabilities,
                              WorkspaceFoldersServerCapabilities, WorkspaceServerCapabilities)
-from pygls.lsp.types.language_features.semantic_tokens import (SemanticTokensLegend,
-                                                               SemanticTokensOptions)
 
 
 class ServerCapabilitiesBuilder:
@@ -229,15 +231,40 @@ def _with_call_hierarchy(self):
         return self
 
     def _with_semantic_tokens(self):
-        value = self._provider_options(TEXT_DOCUMENT_SEMANTIC_TOKENS,
-                                       default=SemanticTokensOptions(
-                                           legend=SemanticTokensLegend(
-                                               token_types=[],
-                                               token_modifiers=[],
-                                           ),
-                                       ))
-        if value is not None:
+
+        providers = [
+            TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+            TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL_DELTA,
+            TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE
+        ]
+
+        for provider in providers:
+            value = self._provider_options(provider, None)
+            if value:
+                break
+
+        if value is None:
+            return self
+
+        if isinstance(value, SemanticTokensRegistrationOptions):
             self.server_cap.semantic_tokens_provider = value
+            return self
+
+        full_support = (
+            SemanticTokensRequestsFull(delta=True)
+            if TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL_DELTA in self.features
+            else TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL in self.features
+        )
+
+        options = SemanticTokensOptions(
+            legend=value,
+            full=full_support or None,
+            range=TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE in self.features or None
+        )
+
+        if options.full or options.range:
+            self.server_cap.semantic_tokens_provider = options
+
         return self
 
     def _with_linked_editing_range(self):
diff --git a/pygls/lsp/__init__.py b/pygls/lsp/__init__.py
@@ -108,17 +108,17 @@
         Optional[List[Moniker]],
     ),
     TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL: (
-        Union[SemanticTokensOptions, SemanticTokensRegistrationOptions],
+        Union[SemanticTokensLegend, SemanticTokensRegistrationOptions],
         SemanticTokensParams,
         Union[SemanticTokensPartialResult, Optional[SemanticTokens]],
     ),
     TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL_DELTA: (
-        Union[SemanticTokensOptions, SemanticTokensRegistrationOptions],
+        Union[SemanticTokensLegend, SemanticTokensRegistrationOptions],
         SemanticTokensDeltaParams,
         Union[SemanticTokensDeltaPartialResult, Optional[Union[SemanticTokens, SemanticTokensDelta]]],
     ),
     TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE: (
-        Union[SemanticTokensOptions, SemanticTokensRegistrationOptions],
+        Union[SemanticTokensLegend, SemanticTokensRegistrationOptions],
         SemanticTokensRangeParams,
         Union[SemanticTokensPartialResult, Optional[SemanticTokens]],
 
diff --git a/tests/lsp/test_semantic_tokens.py b/tests/lsp/test_semantic_tokens.py
