Expose function for computing non-encoded semantic tokens

...this will be useful for testing

Author: fwcd

server/src/main/kotlin/org/javacs/kt/KotlinTextDocumentService.kt

@@ -14,7 +14,7 @@ import org.javacs.kt.position.offset
 import org.javacs.kt.position.extractRange
 import org.javacs.kt.position.position
 import org.javacs.kt.references.findReferences
-import org.javacs.kt.semantictokens.semanticTokens
+import org.javacs.kt.semantictokens.encodedSemanticTokens
 import org.javacs.kt.signaturehelp.fetchSignatureHelpAt
 import org.javacs.kt.symbols.documentSymbols
 import org.javacs.kt.util.noResult
@@ -233,7 +233,7 @@ class KotlinTextDocumentService(
             val uri = parseURI(params.textDocument.uri)
             val file = sp.currentVersion(uri)
 
-            val tokens = semanticTokens(file)
+            val tokens = encodedSemanticTokens(file)
             LOG.info("Found {} tokens", tokens.size)
 
             SemanticTokens(tokens)
@@ -247,7 +247,7 @@ class KotlinTextDocumentService(
             val uri = parseURI(params.textDocument.uri)
             val file = sp.currentVersion(uri)
 
-            val tokens = semanticTokens(file, params.range)
+            val tokens = encodedSemanticTokens(file, params.range)
             LOG.info("Found {} tokens", tokens.size)
 
             SemanticTokens(tokens)

server/src/main/kotlin/org/javacs/kt/semantictokens/SemanticTokens.kt

@@ -26,7 +26,7 @@ import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiNameIdentifierOwner
 import com.intellij.openapi.util.TextRange
 
-private enum class SemanticTokenType(val typeName: String) {
+enum class SemanticTokenType(val typeName: String) {
     KEYWORD(SemanticTokenTypes.Keyword),
     VARIABLE(SemanticTokenTypes.Variable),
     FUNCTION(SemanticTokenTypes.Function),
@@ -39,7 +39,7 @@ private enum class SemanticTokenType(val typeName: String) {
     TYPE(SemanticTokenTypes.Type)
 }
 
-private enum class SemanticTokenModifier(val modifierName: String) {
+enum class SemanticTokenModifier(val modifierName: String) {
     DECLARATION(SemanticTokenModifiers.Declaration),
     DEFINITION(SemanticTokenModifiers.Definition),
     ABSTRACT(SemanticTokenModifiers.Abstract),
@@ -51,14 +51,21 @@ val semanticTokensLegend = SemanticTokensLegend(
     SemanticTokenModifier.values().map { it.modifierName }
 )
 
-private data class SemanticToken(val range: Range, val type: SemanticTokenType, val modifiers: Set<SemanticTokenModifier> = setOf())
+data class SemanticToken(val range: Range, val type: SemanticTokenType, val modifiers: Set<SemanticTokenModifier> = setOf())
+
+/**
+ * Computes LSP-encoded semantic tokens for the given range in the
+ * document. No range means the entire document.
+ */
+fun encodedSemanticTokens(file: CompiledFile, range: Range? = null): List<Int> =
+    encodeTokens(semanticTokens(file, range))
 
 /**
  * Computes semantic tokens for the given range in the document.
  * No range means the entire document.
  */
-fun semanticTokens(file: CompiledFile, range: Range? = null): List<Int> =
-    encodeTokens(elementTokens(file.parse, file.compile, range))
+fun semanticTokens(file: CompiledFile, range: Range? = null): Sequence<SemanticToken> =
+    elementTokens(file.parse, file.compile, range)
 
 private fun encodeTokens(tokens: Sequence<SemanticToken>): List<Int> {
     val encoded = mutableListOf<Int>()