feat: introduce ToonIndentingLexer and refactor

Author: xepozz

src/main/kotlin/com/github/xepozz/toon/language/ToonParserDefinition.kt

@@ -1,17 +1,16 @@
 package com.github.xepozz.toon.language
 
-import com.github.xepozz.toon.language.parser.ToonLexerAdapter
+import com.github.xepozz.toon.language.parser.ToonIndentingLexer
 import com.github.xepozz.toon.language.parser.ToonParser
 import com.github.xepozz.toon.language.psi.ToonTokenSets
 import com.github.xepozz.toon.language.psi.ToonTypes
 import com.intellij.lang.ASTNode
 import com.intellij.lang.ParserDefinition
 import com.intellij.psi.FileViewProvider
 import com.intellij.psi.tree.IFileElementType
-import com.intellij.psi.tree.TokenSet
 
 class ToonParserDefinition : ParserDefinition {
-    override fun createLexer(project: com.intellij.openapi.project.Project?) = ToonLexerAdapter()
+    override fun createLexer(project: com.intellij.openapi.project.Project?) = ToonIndentingLexer()
 
     override fun getWhitespaceTokens() = TokenSet.WHITE_SPACE
 

src/main/kotlin/com/github/xepozz/toon/language/ToonSyntaxHighlighter.kt

@@ -1,6 +1,6 @@
 package com.github.xepozz.toon.language
 
-import com.github.xepozz.toon.language.parser.ToonLexerAdapter
+import com.github.xepozz.toon.language.parser.ToonLexer
 import com.github.xepozz.toon.language.psi.ToonTypes
 import com.intellij.openapi.editor.DefaultLanguageHighlighterColors
 import com.intellij.openapi.editor.HighlighterColors
@@ -10,7 +10,7 @@ import com.intellij.psi.TokenType
 import com.intellij.psi.tree.IElementType
 
 class ToonSyntaxHighlighter : SyntaxHighlighterBase() {
-    override fun getHighlightingLexer() = ToonLexerAdapter()
+    override fun getHighlightingLexer() = ToonLexer()
 
     override fun getTokenHighlights(tokenType: IElementType) = when (tokenType) {
         ToonTypes.TEXT -> STRING_KEYS

…toon/language/parser/ToonLexerAdapter.kt …language/parser/IndentingLexerAdapter.ktsrc/main/kotlin/com/github/xepozz/toon/language/parser/ToonLexerAdapter.kt renamed to src/main/kotlin/com/github/xepozz/toon/language/parser/IndentingLexerAdapter.kt src/main/kotlin/com/github/xepozz/toon/language/parser/ToonLexerAdapter.kt renamed to src/main/kotlin/com/github/xepozz/toon/language/parser/IndentingLexerAdapter.kt

@@ -1,34 +1,31 @@
 package com.github.xepozz.toon.language.parser
 
-import com.github.xepozz.toon.language.psi.ToonTypes
-import com.intellij.lexer.FlexAdapter
 import com.intellij.lexer.Lexer
 import com.intellij.lexer.LookAheadLexer
-
-class ToonLexerAdapter : LookAheadLexer(FlexAdapter(ToonLexer(null))) {
+import com.intellij.psi.tree.IElementType
+
+abstract class IndentingLexerAdapter(
+    lexer: Lexer,
+    private val eolTokenType: IElementType,
+    private val indentTokenType: IElementType,
+    private val dedentTokenType: IElementType,
+) : LookAheadLexer(lexer) {
     private val indentStack = mutableListOf(0)
-    private var start = false
 
     override fun lookAhead(baseLexer: Lexer) {
         val tokenType = baseLexer.tokenType
 
-        println("tokenType: $tokenType")
         if (tokenType == null) {
-            if (!start) {
-                start = true
-                super.lookAhead(baseLexer)
-                return
-            }
             val endOffset = baseLexer.tokenEnd
-            println("compensation last: ${indentStack.size}, position: $endOffset")
             while (indentStack.size > 1) {
                 indentStack.removeLast()
-                addToken(endOffset, ToonTypes.DEDENT)
+                addToken(endOffset, dedentTokenType)
             }
+            advanceAs(baseLexer, tokenType)
             return
         }
 
-        if (tokenType != ToonTypes.EOL) {
+        if (tokenType != eolTokenType) {
             advanceAs(baseLexer, tokenType)
             return
         }
@@ -69,16 +66,16 @@ class ToonLexerAdapter : LookAheadLexer(FlexAdapter(ToonLexer(null))) {
         when {
             newIndent > currentIndent -> {
                 indentStack.add(newIndent)
-                addToken(logicalLineStart, ToonTypes.INDENT)
+                addToken( indentTokenType)
             }
 
             newIndent < currentIndent -> {
-                println("compensation in process: ${indentStack.size}, position: $logicalLineStart")
                 while (indentStack.size > 1 && newIndent < indentStack.last()) {
                     indentStack.removeLast()
-                    addToken(logicalLineStart, ToonTypes.DEDENT)
+                    addToken(logicalLineStart, dedentTokenType)
                 }
             }
+
             else -> {
                 // Same indent – nothing to emit
             }

src/main/kotlin/com/github/xepozz/toon/language/parser/Toon.flex

@@ -6,7 +6,7 @@ import com.intellij.psi.TokenType;
 import com.github.xepozz.toon.language.psi.ToonTypes;
 
 %%
-%class ToonLexer
+%class _ToonLexer
 %implements FlexLexer
 %unicode
 %function advance

src/main/kotlin/com/github/xepozz/toon/language/parser/ToonIndentingLexer.kt

@@ -0,0 +1,10 @@
+package com.github.xepozz.toon.language.parser
+
+import com.github.xepozz.toon.language.psi.ToonTypes
+
+class ToonIndentingLexer : IndentingLexerAdapter(
+    ToonLexer(),
+    ToonTypes.EOL,
+    ToonTypes.INDENT,
+    ToonTypes.DEDENT,
+)

src/main/kotlin/com/github/xepozz/toon/language/parser/ToonLexer.kt

@@ -0,0 +1,6 @@
+package com.github.xepozz.toon.language.parser
+
+import com.intellij.lexer.FlexAdapter
+
+class ToonLexer : FlexAdapter(_ToonLexer(null))
+