feat: implement indent handling in lexer and parser

Author: xepozz

playground/simple1.toon

@@ -0,0 +1 @@
+key: value

playground/simple2.toon

@@ -0,0 +1,2 @@
+key1: value1
+key2: value2

playground/simple3.toon

@@ -0,0 +1,3 @@
+key1:
+    - value1
+    - value2

playground/simple4.toon

@@ -0,0 +1,7 @@
+key1:
+    - value1
+    - value2
+
+key2:
+    - value1
+    - value2

src/main/kotlin/com/github/xepozz/toon/language/ToonParserDefinition.kt

@@ -16,6 +16,7 @@ class ToonParserDefinition : ParserDefinition {
     override fun getWhitespaceTokens() = TokenSet.WHITE_SPACE
 
     override fun createParser(project: com.intellij.openapi.project.Project?) = ToonParser()
+//    override fun createParser(project: com.intellij.openapi.project.Project?) = YAMLParser()
 
     override fun getFileNodeType() = FILE
 

src/main/kotlin/com/github/xepozz/toon/language/ToonSyntaxHighlighter.kt

@@ -15,8 +15,7 @@ class ToonSyntaxHighlighter : SyntaxHighlighterBase() {
     override fun getTokenHighlights(tokenType: IElementType) = when (tokenType) {
         ToonTypes.TEXT -> STRING_KEYS
         ToonTypes.NUMBER -> NUMBER_KEYS
-        ToonTypes.TRUE -> CONSTANT_KEYS
-        ToonTypes.FALSE -> CONSTANT_KEYS
+        ToonTypes.BOOLEAN -> CONSTANT_KEYS
         ToonTypes.NULL -> CONSTANT_KEYS
         ToonTypes.COMMENT -> COMMENT_KEYS
         TokenType.BAD_CHARACTER -> BAD_CHAR_KEYS

src/main/kotlin/com/github/xepozz/toon/language/parser/Toon.bnf

@@ -4,57 +4,101 @@
   extends = "com.intellij.extapi.psi.ASTWrapperPsiElement"
 
   psiClassPrefix = "Toon"
-  psiImplClassSuffix="Impl"
-  psiPackage="com.github.xepozz.toon.language.psi"
-  psiImplPackage="com.github.xepozz.toon.language.psi.impl"
+  psiImplClassSuffix = "Impl"
+  psiPackage = "com.github.xepozz.toon.language.psi"
+  psiImplPackage = "com.github.xepozz.toon.language.psi.impl"
 
-  elementTypeHolderClass="com.github.xepozz.toon.language.psi.ToonTypes"
+  elementTypeHolderClass = "com.github.xepozz.toon.language.psi.ToonTypes"
   elementTypeClass = "com.github.xepozz.toon.language.psi.ToonElementType"
   tokenTypeClass = "com.github.xepozz.toon.language.psi.ToonTokenType"
 
-  psiImplUtilClass="com.github.xepozz.toon.language.psi.impl.ToonImplUtil"
-  tokens=[
-    LPAREN="LPAREN"
-    RPAREN="RPAREN"
+  psiImplUtilClass = "com.github.xepozz.toon.language.psi.impl.ToonImplUtil"
+
+  tokens = [
+    LPAREN = "LPAREN"
+    RPAREN = "RPAREN"
+    LBRACE = "LBRACE"
+    RBRACE = "RBRACE"
+    LBRACKET = "LBRACKET"
+    RBRACKET = "RBRACKET"
+    DELIMITER = "DELIMITER"
+    COMMA = "COMMA"
+    DASH = "DASH"
+    SHARP = "SHARP"
+    EOL = "EOL"
+    COMMENT = "COMMENT"
+    TEXT = "TEXT"
+    NUMBER = "NUMBER"
+    NULL = "NULL"
+    BOOLEAN = "BOOLEAN"
+    INDENT = "INDENT"
+    DEDENT = "DEDENT"
+    BLOCK = "BLOCK"
   ]
 }
 
 file ::= entry*
 
-private entry ::= table | keyValueMap | COMMENT | EOL
+private entry ::=
+    object_field
+  | table
+  | list_item
+  | COMMENT
+  | EOL
+object_field ::= key DELIMITER field_value {
+  pin = 2
+}
+private field_value ::=
+    value_list
+  | nested_object
+  | array
 
-table ::= tableDefinition DELIMITER (iterable | EOL array)
-{
-    pin=2
+value_list ::= value (COMMA value)*
+
+nested_object ::= EOL INDENT entry+ DEDENT {
+    pin = 2
 }
 
-tableDefinition ::= key LBRACKET SHARP? NUMBER RBRACKET (LBRACE iterable RBRACE)?
-{
-    pin=2
+array ::= list_item (EOL list_item)+ {
+    pin = 1
 }
 
-array ::= keyValueArray+ | keyValueMap+ | valuesList
+list_item ::= DASH list_item_content {
+  pin = 1
+}
 
-valuesList ::= iterable (EOL iterable)*
+private list_item_content ::=
+    value_list
+  | key DELIMITER field_value
 
-keyValueArray ::= keyValueArrayItem (EOL keyValueArrayItem)*
+table ::= table_header DELIMITER table_data EOL {
+  pin = 2
+}
 
-keyValueArrayItem ::= DASH (keyValueMap | iterable)
-{
-    pin=1
+table_header ::= key LBRACKET table_size RBRACKET field_list? {
+    pin=4
 }
 
-keyValueMap ::= key DELIMITER (iterable | EOL array)
-{
-    pin=2
+table_size ::= SHARP? NUMBER
+
+field_list ::= LBRACE field_names RBRACE {
+  pin = 1
 }
 
-key ::= TEXT
+field_names ::= key (COMMA key)*
 
-iterable ::= value (COMMA value)*
+private table_data ::=
+    table_row | (EOL table_row)+
 
-value ::= TEXT | NULL | FALSE | TRUE | NUMBER
-{
-    implements=["com.intellij.psi.PsiLiteralValue"]
-    methods=[getValue getReferences]
-}
+table_row ::= value (COMMA value)*
+
+key ::= TEXT
+
+value ::=
+    TEXT
+  | NUMBER
+  | NULL
+  | BOOLEAN {
+    implements = ["com.intellij.psi.PsiLiteralValue"]
+    methods = [getValue getReferences]
+  }

src/main/kotlin/com/github/xepozz/toon/language/parser/Toon.flex

@@ -14,14 +14,19 @@ import com.github.xepozz.toon.language.psi.ToonTypes;
 %eof{  return;
 %eof}
 
+%{
+private int prevIndent = 0;
+private boolean atLineStart = true;
+%}
 
-NEWLINE=\r|\n|\r\n
+
+NEWLINE=\r\n|\r|\n
 WHITESPACE=[ \t]+
 ALPHA=[A-Za-z]
 NUMBER=[\d]+(.[\d]+)?
 QUOTTED_STRING = "\""(\\\"|[^\"])*"\"" | "\'"(\\\'|[^\'])*"\'"
 
-%state VALUE, TABLE_SIZE
+%state VALUE, TABLE_SIZE, MAIN
 %%
 
 <YYINITIAL> {
@@ -34,17 +39,17 @@ QUOTTED_STRING = "\""(\\\"|[^\"])*"\"" | "\'"(\\\'|[^\'])*"\'"
 <TABLE_SIZE> {
     {NUMBER}          { return ToonTypes.NUMBER; }
     "#"               { return ToonTypes.SHARP; }
-    "]"               { yybegin(YYINITIAL); return ToonTypes.RBRACKET; }
+    "]"               { yybegin(MAIN); return ToonTypes.RBRACKET; }
 }
 
 <VALUE> {
     "null"                  { return ToonTypes.NULL; }
-    "false"                 { return ToonTypes.FALSE; }
-    "true"                  { return ToonTypes.TRUE; }
+    "false"|"true"          { return ToonTypes.BOOLEAN; }
     ","                     { return ToonTypes.COMMA; }
     {NUMBER}                { return ToonTypes.NUMBER; }
     {ALPHA}[^\n,]*          { return ToonTypes.TEXT; }
     {QUOTTED_STRING}        { return ToonTypes.TEXT; }
+    {WHITESPACE}            { return TokenType.WHITE_SPACE; }
 }
 
 
@@ -58,6 +63,6 @@ QUOTTED_STRING = "\""(\\\"|[^\"])*"\"" | "\'"(\\\'|[^\'])*"\'"
 "]"                     { return ToonTypes.RBRACKET; }
 "-"                     { return ToonTypes.DASH; }
 {WHITESPACE}            { return TokenType.WHITE_SPACE; }
-{NEWLINE}               { yybegin(YYINITIAL); return ToonTypes.EOL; }
+{NEWLINE}               { atLineStart = true; yybegin(YYINITIAL); return ToonTypes.EOL; }
 
 [^]                     { return TokenType.BAD_CHARACTER; }

src/main/kotlin/com/github/xepozz/toon/language/parser/ToonLexerAdapter.kt

@@ -1,5 +1,73 @@
 package com.github.xepozz.toon.language.parser
 
+import com.github.xepozz.toon.language.psi.ToonTypes
 import com.intellij.lexer.FlexAdapter
+import com.intellij.lexer.Lexer
+import com.intellij.lexer.LookAheadLexer
 
-class ToonLexerAdapter : FlexAdapter(ToonLexer(null))
+class ToonLexerAdapter : LookAheadLexer(FlexAdapter(ToonLexer(null))) {
+    private val indentStack = mutableListOf(0)
+
+    override fun lookAhead(baseLexer: Lexer) {
+        val tokenType = baseLexer.tokenType
+
+        if (tokenType == null) {
+            addToken(ToonTypes.DEDENT)
+            super.lookAhead(baseLexer)
+            return
+        }
+
+        if (tokenType != ToonTypes.EOL) {
+            advanceAs(baseLexer, tokenType)
+            return
+        }
+
+        // Emit the newline token
+        advanceAs(baseLexer, tokenType)
+
+        // Inspect the beginning of the next line
+        val buffer = baseLexer.bufferSequence
+        val bufferEnd = baseLexer.bufferEnd
+
+        var index = baseLexer.tokenStart
+        if (index >= bufferEnd) return
+
+        var columns = 0
+
+        // Count leading spaces
+        while (index < bufferEnd) {
+            val c = buffer[index]
+            when (c) {
+                ' ' -> {
+                    columns++
+                    index++
+                }
+                '\r', '\n' -> return
+                else -> break
+            }
+        }
+
+        if (index >= bufferEnd) return
+
+        val newIndent = columns
+        val currentIndent = indentStack.last()
+
+        val logicalLineStart = index // Actual content starts here
+
+        when {
+            newIndent > currentIndent -> {
+                indentStack.add(newIndent)
+                addToken(ToonTypes.INDENT)
+            }
+            newIndent < currentIndent -> {
+                while (indentStack.size > 1 && newIndent < indentStack.last()) {
+                    indentStack.removeAt(indentStack.size - 1)
+                    addToken(logicalLineStart, ToonTypes.DEDENT)
+                }
+            }
+            else -> {
+                // Same indent – nothing to emit
+            }
+        }
+    }
+}

src/main/resources/META-INF/plugin.xml

@@ -11,7 +11,7 @@
 
     <extensions defaultExtensionNs="com.intellij">
         <fileType
-                name="TOON file"
+                name="TOON"
                 implementationClass="com.github.xepozz.toon.language.ToonFileType"
                 language="TOON"
                 patterns="*.toon;*.toon.yaml"