Implement fixed width lexing

Author: tecosaur

README.org

@@ -53,7 +53,7 @@ TODO
 | DiarySexp           |      | X   |       |     |      |      |
 | Planning            |      | X   |       |     |      |      |
 | Comment             |      | X   |       |     |      |      |
-| FixedWidth          |      |     |       |     |      |      |
+| FixedWidth          |      | X   |       |     |      |      |
 | HorizontalRule      |      |     |       |     |      |      |
 | Keyword             |      | X   |       |     |      |      |
 | Affiliated Keywords |      | -   |       |     |      |      |

src/lexer.jl

@@ -93,7 +93,11 @@ function lexnext(state::LexerState, bytes::DenseVector{UInt8}, start::UInt32)::T
         elseif chr == UInt8('*') && pos == linestart && ischarat(bytes, pos + countsame(bytes, pos, '*'), ' ')
             lex_heading(state, bytes, pos)
         elseif chr == UInt8(':')
-            lex_drawer(state, bytes, pos)
+            if length(bytes) > pos && iswhitespace(bytes, pos + 0x1) || islineend(bytes, pos + 0x1)
+                lex_fixedwidth(state, bytes, pos)
+            else
+                lex_drawer(state, bytes, pos)
+            end
         elseif chr == UInt8('[') && pos == linestart
             fndef = lex_footnotedef(state, bytes, pos)
             if fndef != NONE_TOKEN && K"footnote_definition" ∈ state.ctx
@@ -428,7 +432,19 @@ function lex_comment(::LexerState, bytes::DenseVector{UInt8}, start::UInt32)
     Token(K"comment", start, pos), nextpos
 end
 
-# TODO: Fixed width
+function lex_fixedwidth(::LexerState, bytes::DenseVector{UInt8}, start::UInt32)
+    pos, nextpos = start, start
+    while pos <= length(bytes)
+        nextpos = skipspaces(bytes, nextpos).stop
+        ischarat(bytes, nextpos, ':') &&
+            (islineend(bytes, nextpos + 0x1) ||
+             nextpos < length(bytes) && iswhitespace(bytes, nextpos + 0x1)) ||
+            break
+        pos = lineend(bytes, nextpos) - 0x1
+        nextpos = pos + 0x2
+    end
+    Token(K"fixedwidth", start, pos), nextpos
+end
 
 # TODO: Horizontal rules
 

test/runtests.jl

@@ -266,6 +266,18 @@ end
         @test collect(Lexer("# comment\n # \n# more\n")) ==
             [Token(K"comment", 1, 20)]
     end
+    @testset "Fixed width" begin
+        @test collect(Lexer(":")) ==
+            [Token(K"fixedwidth", 1, 1)]
+        @test collect(Lexer(": fixed")) ==
+            [Token(K"fixedwidth", 1, 7)]
+        @test collect(Lexer(": fixed\n: more")) ==
+            [Token(K"fixedwidth", 1, 14)]
+        @test collect(Lexer(": fixed\n:\n: more")) ==
+            [Token(K"fixedwidth", 1, 16)]
+        @test collect(Lexer(": fixed\n : \n: more\n")) ==
+            [Token(K"fixedwidth", 1, 18)]
+    end
     @testset "Type inference" begin
         @testset "Utilities" begin
             bytes, pos = codeunits("abc"), UInt32(1)
@@ -302,6 +314,7 @@ end
             @inferred Tuple{Token, UInt32} Org.lex_diarysexp(lstate, bytes, pos)
             @inferred Tuple{Token, UInt32} Org.lex_planning(lstate, bytes, pos)
             @inferred Tuple{Token, UInt32} Org.lex_comment(lstate, bytes, pos)
+            @inferred Tuple{Token, UInt32} Org.lex_fixedwidth(lstate, bytes, pos)
         end
     end
     @testset "Unhandled errors" begin
@@ -340,6 +353,7 @@ end
             @test_call Org.lex_diarysexp(lstate, bytes, pos)
             @test_call Org.lex_planning(lstate, bytes, pos)
             @test_call Org.lex_comment(lstate, bytes, pos)
+            @test_call Org.lex_fixedwidth(lstate, bytes, pos)
         end
         @testset "Iteration" begin
             @test_call iterate(Lexer("abc"), LexerState())
@@ -381,6 +395,7 @@ end
             @test_opt Org.lex_diarysexp(lstate, bytes, pos)
             @test_opt Org.lex_planning(lstate, bytes, pos)
             @test_opt Org.lex_comment(lstate, bytes, pos)
+            @test_opt Org.lex_fixedwidth(lstate, bytes, pos)
         end
         @testset "Iteration" begin
             @test_opt iterate(Lexer("abc"), LexerState())