Fix parsing of indented modules

Indented modules weren't parsing due to an error in how the recovery
routine derived the current indent column.

Recovery derived the indent from the layout stack in the token stream.
This layout stack, however, represents the stack after the current
token. This is usually OK, but has an edge case when the current token
is TokLayoutEnd. For the purposes of recovery, we want TokLayoutEnd's
column to represent the current indent.

In the specific case of top-level module/declaration parsing, we invoke
recovery for all declarations, which are also wrapped in a `many`.
Indentation recovery eats any token indented more than the current
indentation. Once we get to a TokLayoutEnd for the module, which closes
the top-level `where` block, the current layout stack is only LytRoot,
so the indent defaults to 0. Because the module is also indented to 0,
this will fail the recovery predicate as we expect, resulting in a
correct module parse. When the indent is more than 0, it will satisfy
the recovery predicate (since TokLayoutEnd col is greater than 0),
eating the TokLayoutEnd token. Module parsing would then fail due to an
unexpected EOF (where TokLayoutEnd was expected).

When take the TokLayoutEnd column into account when deriving the
current indent, then the recovery predicate will fail as expected,
resulting in a correct parse.

Fixes #56

Author: natefaubion

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
         run: npm run install
 
       - name: Build project
-        run: npm run build -- --pedantic-packages
+        run: npm run build -- --pedantic-packages --strict
 
       - name: Run tests
         run: npm run test -- --offline --quiet

bench/src/ParseFile.purs

@@ -17,7 +17,7 @@ import Node.FS.Aff (readFile)
 import Node.Process as Process
 import PureScript.CST (RecoveredParserResult(..), parseModule)
 import PureScript.CST.Errors (ParseError, printParseError)
-import PureScript.CST.Lexer (lex)
+import PureScript.CST.Lexer (lexModule)
 import PureScript.CST.Parser.Monad (PositionedError)
 import PureScript.CST.Print (TokenOption(..), printSourceTokenWithOption)
 import PureScript.CST.TokenStream (TokenStep(..), TokenStream, step)
@@ -36,7 +36,7 @@ main = launchAff_ do
           tokens =
             map (foldMap (printSourceTokenWithOption ShowLayout))
               $ tokenStreamToArray
-              $ lex contents
+              $ lexModule contents
         for_ tokens Console.log
       else
         mempty

spago.yaml

@@ -1,8 +1,6 @@
 package:
   name: language-cst-parser
   description: A parser for the PureScript concrete syntax tree.
-  build:
-    strict: true
   publish:
     version: 0.14.1
     license: MIT

src/PureScript/CST/Parser.purs

@@ -19,15 +19,14 @@ import Data.Array as Array
 import Data.Array.NonEmpty (NonEmptyArray)
 import Data.Array.NonEmpty as NonEmptyArray
 import Data.Either (Either(..))
-import Data.Maybe (Maybe(..), maybe)
+import Data.Maybe (Maybe(..))
 import Data.Set (Set)
 import Data.Set as Set
 import Data.Tuple (Tuple(..), uncurry)
 import Prim as P
 import PureScript.CST.Errors (ParseError(..), RecoveredError(..))
-import PureScript.CST.Layout (currentIndent)
 import PureScript.CST.Parser.Monad (Parser, eof, lookAhead, many, optional, recover, take, try)
-import PureScript.CST.TokenStream (TokenStep(..), TokenStream, layoutStack)
+import PureScript.CST.TokenStream (TokenStep(..), TokenStream, currentIndentColumn)
 import PureScript.CST.TokenStream as TokenStream
 import PureScript.CST.Types (AppSpine(..), Binder(..), ClassFundep(..), DataCtor(..), DataMembers(..), Declaration(..), Delimited, DoStatement(..), Export(..), Expr(..), Fixity(..), FixityOp(..), Foreign(..), Guarded(..), GuardedExpr(..), Ident(..), Import(..), ImportDecl(..), Instance(..), InstanceBinding(..), IntValue(..), Label(..), Labeled(..), LetBinding(..), Module(..), ModuleBody(..), ModuleHeader(..), ModuleName(..), Name(..), OneOrDelimited(..), Operator(..), PatternGuard(..), Prefixed(..), Proper(..), QualifiedName(..), RecordLabeled(..), RecordUpdate(..), Role(..), Row(..), Separated(..), SourceToken, Token(..), Type(..), TypeVarBinding(..), Where(..), Wrapped(..))
 
@@ -1188,7 +1187,7 @@ recoverTokensWhile :: (SourceToken -> Int -> Boolean) -> TokenStream -> Tuple (A
 recoverTokensWhile p initStream = go [] initStream
   where
   indent :: Int
-  indent = maybe 0 _.column $ currentIndent $ layoutStack initStream
+  indent = currentIndentColumn initStream
 
   go :: Array SourceToken -> TokenStream -> Tuple (Array SourceToken) TokenStream
   go acc stream = case TokenStream.step stream of

src/PureScript/CST/TokenStream.purs

@@ -5,6 +5,7 @@ module PureScript.CST.TokenStream
   , consTokens
   , layoutStack
   , unwindLayout
+  , currentIndentColumn
   ) where
 
 import Prelude
@@ -13,11 +14,11 @@ import Data.Foldable (class Foldable, foldr)
 import Data.Lazy (Lazy)
 import Data.Lazy as Lazy
 import Data.List (List(..), (:))
-import Data.Maybe (Maybe)
+import Data.Maybe (Maybe, maybe)
 import Data.Newtype (class Newtype, unwrap)
 import Data.Tuple (Tuple(..))
 import PureScript.CST.Errors (ParseError)
-import PureScript.CST.Layout (LayoutDelim(..), LayoutStack, isIndented, lytToken)
+import PureScript.CST.Layout (LayoutDelim(..), LayoutStack, currentIndent, isIndented, lytToken)
 import PureScript.CST.Types (Comment, LineFeed, SourcePos, SourceToken, Token(..))
 
 newtype TokenStream = TokenStream (Lazy TokenStep)
@@ -64,3 +65,21 @@ unwindLayout pos eof = go
               TokenCons (lytToken pos (TokLayoutEnd pos'.column)) pos (go tl) tl
           | otherwise ->
               step (go tl)
+
+-- In the token stream, the layout stack represents the state after the token.
+-- When determining the current indent level, this creates an edge case relating
+-- to TokLayoutEnd. The layout stack will return the next indent, but for the
+-- purposes of recovery, we want TokLayoutEnd column to be included as the current
+-- indent, necessitating special handling.
+currentIndentColumn :: TokenStream -> Int
+currentIndentColumn stream = case step stream of
+  TokenError _ _ _ stk ->
+    stkColumn stk
+  TokenEOF _ _ ->
+    0
+  TokenCons { value: TokLayoutEnd col } _ _ _ ->
+    col
+  TokenCons _ _ _ stk ->
+    stkColumn stk
+  where
+  stkColumn = maybe 0 _.column <<< currentIndent

test/Main.purs

@@ -1,4 +1,4 @@
-module Test.Main where
+module Test.Main (main) where
 
 import Prelude
 import Prim hiding (Type)
@@ -370,3 +370,14 @@ main = do
         true
       _ ->
         false
+
+  assertParse "Indented module"
+    """
+    module Test where
+      test = 42
+    """
+    case _ of
+      ParseSucceeded (Module _) ->
+        true
+      _ ->
+        false