Implement semantic tokens test

Author: fwcd

server/src/main/kotlin/org/javacs/kt/semantictokens/SemanticTokens.kt

@@ -80,7 +80,7 @@ fun encodedSemanticTokens(file: CompiledFile, range: Range? = null): List<Int> =
 fun semanticTokens(file: CompiledFile, range: Range? = null): Sequence<SemanticToken> =
     elementTokens(file.parse, file.compile, range)
 
-private fun encodeTokens(tokens: Sequence<SemanticToken>): List<Int> {
+fun encodeTokens(tokens: Sequence<SemanticToken>): List<Int> {
     val encoded = mutableListOf<Int>()
     var last: SemanticToken? = null
 
@@ -149,19 +149,17 @@ private fun elementToken(element: PsiElement, bindingContext: BindingContext): S
         }
         is PsiNameIdentifierOwner -> {
             val tokenType = when (element) {
-                is KtProperty -> SemanticTokenType.PROPERTY
                 is KtParameter -> SemanticTokenType.PARAMETER
+                is KtProperty -> SemanticTokenType.PROPERTY
                 is KtVariableDeclaration -> SemanticTokenType.VARIABLE
                 is KtClassOrObject -> SemanticTokenType.CLASS
                 else -> return null
             }
             val identifierRange = element.nameIdentifier?.let { range(file.text, it.textRange) } ?: return null
             val modifiers = mutableSetOf(SemanticTokenModifier.DECLARATION)
 
-            if (element is KtVariableDeclaration) {
-                if (!element.isVar() || element.hasModifier(KtTokens.CONST_KEYWORD)) {
-                    modifiers.add(SemanticTokenModifier.READONLY)
-                }
+            if (element is KtVariableDeclaration && (!element.isVar() || element.hasModifier(KtTokens.CONST_KEYWORD)) || element is KtParameter) {
+                modifiers.add(SemanticTokenModifier.READONLY)
             }
 
             if (element is KtModifierListOwner) {

server/src/test/kotlin/org/javacs/kt/LanguageServerTestFixture.kt

@@ -68,6 +68,9 @@ abstract class LanguageServerTestFixture(relativeWorkspaceRoot: String) : Langua
     fun hoverParams(relativePath: String, line: Int, column: Int): HoverParams =
         textDocumentPosition(relativePath, line, column).run { HoverParams(textDocument, position) }
 
+    fun semanticTokensParams(relativePath: String): SemanticTokensParams =
+        textDocumentPosition(relativePath, 0, 0).run { SemanticTokensParams(textDocument) }
+
     fun signatureHelpParams(relativePath: String, line: Int, column: Int): SignatureHelpParams =
         textDocumentPosition(relativePath, line, column).run { SignatureHelpParams(textDocument, position) }
 

server/src/test/kotlin/org/javacs/kt/SemanticTokensTest.kt

@@ -3,7 +3,38 @@ package org.javacs.kt
 import org.hamcrest.Matchers.*
 import org.junit.Assert.assertThat
 import org.junit.Test
+import org.javacs.kt.semantictokens.encodeTokens
+import org.javacs.kt.semantictokens.SemanticToken
+import org.javacs.kt.semantictokens.SemanticTokenType
+import org.javacs.kt.semantictokens.SemanticTokenModifier
 
 class SemanticTokensTest : SingleFileTestFixture("semantictokens", "SemanticTokens.kt") {
-    // TODO
+    @Test fun `tokenize entire file`() {
+        val response = languageServer.textDocumentService.semanticTokensFull(semanticTokensParams(file)).get()!!
+        val actual = response.data
+        val expected = encodeTokens(sequenceOf(
+            // SemanticToken(range(1, 5, 1, 13), SemanticTokenType.PROPERTY, setOf(SemanticTokenModifier.DECLARATION)), // variable
+            // SemanticToken(range(2, 5, 2, 13), SemanticTokenType.PROPERTY, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // constant
+            // SemanticToken(range(2, 15, 2, 21), SemanticTokenType.CLASS), // String
+            // SemanticToken(range(2, 24, 2, 40), SemanticTokenType.STRING), // "test $variable"
+            // SemanticToken(range(2, 30, 2, 39), SemanticTokenType.INTERPOLATION_ENTRY), // $variable
+            // SemanticToken(range(2, 31, 2, 39), SemanticTokenType.VARIABLE), // variable
+
+            // SemanticToken(range(4, 12, 4, 16), SemanticTokenType.CLASS, setOf(SemanticTokenModifier.DECLARATION)), // Type
+            // SemanticToken(range(4, 21, 4, 29), SemanticTokenType.PROPERTY, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // property
+            // SemanticToken(range(4, 31, 4, 34), SemanticTokenType.CLASS), // Int
+
+            // SemanticToken(range(6, 5, 6, 6), SemanticTokenType.FUNCTION, setOf(SemanticTokenModifier.DECLARATION)), // f
+            SemanticToken(range(6, 7, 6, 8), SemanticTokenType.PARAMETER, setOf(SemanticTokenModifier.DECLARATION, SemanticTokenModifier.READONLY)), // x
+            SemanticToken(range(6, 10, 6, 13), SemanticTokenType.CLASS), // Int?
+            SemanticToken(range(6, 24, 6, 27), SemanticTokenType.CLASS), // Int
+            SemanticToken(range(6, 30, 6, 31), SemanticTokenType.FUNCTION), // f
+            SemanticToken(range(6, 32, 6, 33), SemanticTokenType.VARIABLE, setOf(SemanticTokenModifier.READONLY)), // x
+        ))
+
+        println(actual)
+        println(expected)
+
+        assertThat(actual, contains(*expected.toTypedArray()))
+    }
 }

server/src/test/resources/semantictokens/SemanticTokens.kt

@@ -0,0 +1,6 @@
+// var variable = 3
+// val constant: String = "test $variable"
+
+// data class Type(val property: Int)
+
+fun f(x: Int? = null): Int = f(x)