Benchmark tweaks, CPS parsing monad, parse fixes

Benchmark only the actual parsing (not opening/reading of source),
and don't add log timing benchmarks if they are severely inflated.
Note that inflated benchmarks can also be caused by doing other stuff
while benchmarking.

This closes #23.

Changed ParseMonad to use a CPS parser. This should let me look to
parsec and attoparsec for performance ideas (since they take the same
approach).

Fixed some parsing bugs:

  * multline strings
  * the string "\\"
  * expressions in statements that start with blocks can be turned
    into regular semicolon terminated expressions if the block ends
    in a postfix operator `(ex: { x }? + 1;`)
  * same point as before, but for block like things in the arms of
    match clauses

This last two turned out to be tricky - I now need to export patterns under a new production
and not export type parameter bounds. I'm not sure why; this all seems arbitrary. To
investigate when I have time...

Author: harpocrates

benchmarks/allocation-benchmarks/Main.hs

@@ -1,12 +1,14 @@
-{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE OverloadedStrings, TypeApplications #-}
 
 import Weigh
 
 import Control.Monad (filterM)
-import Data.Foldable (traverse_)
+import Data.Foldable (for_)
+import Data.Traversable (for)
 import GHC.Exts (fromString)
 
-import Language.Rust.Parser (parseSourceFile')
+import Language.Rust.Syntax (SourceFile)
+import Language.Rust.Parser (readInputStream, Span, parse')
 
 import System.Directory (getCurrentDirectory, listDirectory, createDirectoryIfMissing, doesFileExist)
 import System.FilePath ((</>), (<.>), takeFileName)
@@ -33,7 +35,9 @@ main = do
   files <- filterM doesFileExist entries
 
   -- Run 'weigh' tests
-  let weigh = setColumns [ Case, Max, Allocated, GCs, Live ] >> traverse_ (\f -> io (takeFileName f) parseSourceFile' f) files
+  fileStreams <- for files $ \file -> do { is <- readInputStream file; pure (takeFileName file, is) }
+  let weigh = do setColumns [ Case, Max, Allocated, GCs, Live ]
+                 for_ fileStreams $ \(file,is) -> func file (parse' @(SourceFile Span)) is
   mainWith weigh
   (wr, _) <- weighResults weigh
   let results = object [ case maybeErr of
@@ -50,5 +54,6 @@ main = do
   -- Save the output to JSON
   createDirectoryIfMissing False (workingDirectory </> "allocations")
   let logFile = workingDirectory </> "allocations" </> logFileName <.> "json"
+  putStrLn $ "writing results to: " ++ logFile
   logFile `BL.writeFile` encode results
 

benchmarks/timing-benchmarks/Main.hs

@@ -1,14 +1,16 @@
-{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE OverloadedStrings, TypeApplications #-}
 
 import Criterion
-import Criterion.Types (anMean, reportAnalysis)
+import Criterion.Main (defaultConfig)
+import Criterion.Types (anMean, reportAnalysis, timeLimit, anOutlierVar, ovEffect, OutlierEffect(Severe))
 import Statistics.Resampling.Bootstrap (Estimate(..))
 
 import Control.Monad (filterM)
 import Data.Traversable (for)
 import GHC.Exts (fromString)
 
-import Language.Rust.Parser (parseSourceFile')
+import Language.Rust.Syntax (SourceFile)
+import Language.Rust.Parser (readInputStream, Span, parse')
 
 import System.Directory (getCurrentDirectory, listDirectory, createDirectoryIfMissing, doesFileExist)
 import System.FilePath ((</>), (<.>), takeFileName)
@@ -35,18 +37,21 @@ main = do
   reports <- for files $ \f -> do
     let name = takeFileName f
     putStrLn name
-    bnch <- benchmark' (nfIO (parseSourceFile' f))
+    is <- readInputStream f
+    bnch <- benchmarkWith' defaultConfig{ timeLimit = 15 } (nf (parse' @(SourceFile Span)) is)
     pure (name, bnch)
   let results = object [ fromString name .= object [ "mean" .= m
                                                    , "lower bound" .= l
                                                    , "upper bound" .= u
                                                    ]
                        | (name,report) <- reports
                        , let Estimate m l u _ = anMean (reportAnalysis report)
+                       , ovEffect (anOutlierVar (reportAnalysis report)) /= Severe
                        ]
 
   -- Save the output to JSON
   createDirectoryIfMissing False (workingDirectory </> "timings")
   let logFile = workingDirectory </> "timings" </> logFileName <.> "json"
+  putStrLn $ "writing results to: " ++ logFile
   logFile `BL.writeFile` encode results
 

sample-sources/expressions.rs

@@ -43,6 +43,11 @@ fn main() {
   let x = foo()?;
   return 0;
   return;
+
+  match true {
+    true => move | | { 1 },
+    false => | | { 2}
+  }
 }
 
 fn precedences() {

sample-sources/literals.rs

@@ -18,9 +18,21 @@ fn main() {
 
   // Strings
   "hello world";
+  "hello
+  world";
+  "hello\
+  world";
   r"hello world";
+  r"hello
+  world";
+  r"hello\
+  world";
   r#"hello " world"#;
   b"hello world";
+  b"hello
+  world";
+  b"hello\
+  world";
   br"hello world";
   br#"hello " world"#;
 

sample-sources/precedences.rs

@@ -46,6 +46,14 @@ fn range_expressions() {
   return |x: i32| x + 1;
   box 1 + 2;
   |x| { 1 };
+  x || y();
+  x && y();
+
+  // block expression / starting with block statement
+  { 1 }?.0 + 1;
+  if true { 1 } else { 2 }.toString();
+  if true { 1 }.toString();
+  { x }[2]?.foo * { 3 };
 
   // no struct expression
   for x in 1.. { }

sample-sources/statement-expressions.rs

@@ -0,0 +1,52 @@
+fn main() {
+
+  { 1 }[1];     // !!! Parses as { 1 }; [1];
+  { 1 }(0);     // !!! Parses as { 1 }; (0);
+  
+  { 1 }.foo;    // Parses as a field access
+  { 1 }.foo(0); // Parses as a method call
+  { 1 }.0;      // Parses as a tup field access
+  { 1 }?;       // Parses as a try
+  
+
+  { 1 }? + 1;  // SHOULD WORK
+  { 1 }[0] + 1;  // SHOULD WORK
+  { 1 }(0,1,2) + 1;  // SHOULD WORK
+  { 1 }.foo(0,1,2) + 1;  // SHOULD WORK
+  { 1 }.foo + 1;  // SHOULD WORK
+  { 1 }.0 + 1;  // SHOULD WORK
+
+ // { 1 } as i32 + 1;  // SHOULD NOT WORK
+ // { 1 }  + 1;  // SHOULD NOT WORK
+  
+  { 1 }[1];
+  { 1 }();
+  { 1 }.bar;
+  { 1 }.bar();
+  { 1 }.0;
+
+
+  if true { 1 } [1];
+  if true { 1 } ();
+  if true { 1 } .bar;
+  if true { 1 } .bar();
+
+  if true { 1 } else { 2 }[1];
+  if true { 1 } else { 2 }();
+  if true { 1 } else { 2 }.bar;
+  if true { 1 } else { 2 }.bar();
+
+
+  loop { 1 } [1];
+  loop { 1 } ();
+  loop { 1 } .bar;
+  loop { 1 } .bar();
+
+  match true {
+   // true => { 1 } + 2, // SHOULD NOT WORK
+    true => { 1 }? + 2, // SHOULD WORK
+    false => 1,
+    true => move | | { 1 },
+    false => 1
+  }
+}

src/Language/Rust/Parser.hs

@@ -85,7 +85,6 @@ instance Parse TokenTree where parser = parseTt
 instance Parse (Block Span) where parser = parseBlock
 instance Parse (ImplItem Span) where parser = parseImplItem 
 instance Parse (TraitItem Span) where parser = parseTraitItem
-instance Parse (TyParamBound Span) where parser = parseTyParamBound
 instance Parse (TyParam Span) where parser = parseTyParam
 instance Parse (LifetimeDef Span) where parser = parseLifetimeDef
 instance Parse (Generics Span) where parser = parseGenerics

src/Language/Rust/Parser/Internal.y

@@ -28,7 +28,7 @@ To get information about transition states and such, run
 
 module Language.Rust.Parser.Internal (
   parseLit, parseAttr, parseTy, parsePat, parseStmt, parseExpr, parseItem, parseSourceFile,
-  parseBlock, parseImplItem, parseTraitItem, parseTt, parseTyParamBound, parseTyParam,
+  parseBlock, parseImplItem, parseTraitItem, parseTt, parseTyParam,
   parseGenerics, parseWhereClause, parseLifetimeDef, parseArg
 ) where
 
@@ -48,7 +48,7 @@ import Text.Read (readMaybe)
 %name parseLit lit
 %name parseAttr attribute
 %name parseTy ty_general
-%name parsePat pat
+%name parsePat export_pat
 %name parseArg export_arg
 %name parseStmt stmt
 %name parseExpr expr
@@ -58,7 +58,6 @@ import Text.Read (readMaybe)
 %name parseImplItem impl_item
 %name parseTraitItem trait_item
 %name parseTt token_tree
-%name parseTyParamBound ty_param_bound_mod
 %name parseTyParam ty_param
 %name parseLifetimeDef lifetime_def
 %name parseWhereClause where_clause
@@ -1065,15 +1064,12 @@ comma_arms :: { [Arm Span] }
 
 -- An expression followed by match arms. If there is a comma needed, it is added
 expr_arms :: { (Expr Span, [Arm Span]) }
-  : gen_expression(nonblock_expr,expr,expr) comma_arms  { ($1, $2) }
-  | paren_expr                              comma_arms  { ($1, $2) }
-  | struct_expr                             comma_arms  { ($1, $2) }
+  : nonblock_expr                           comma_arms  { ($1, $2) }
+  | postfix_block_expr                      comma_arms  { ($1, $2) }
+  | vis_safety_block                        comma_arms  { ($1, $2) }
+  | vis_safety_block                              arms  { ($1, $2) }
   | block_like_expr                         comma_arms  { ($1, $2) }
-  | inner_attrs_block                       comma_arms  { let (as,b) = $1 in (BlockExpr as b (spanOf b), $2) }
-  | inner_attrs_block                             arms  { let (as,b) = $1 in (BlockExpr as b (spanOf b), $2) }
-  | unsafe inner_attrs_block                comma_arms
-    { let (as, Block ss r x) = $2 in (BlockExpr as (Block ss Unsafe ($1 # x)) ($1 # x), $3) }
-
+  | block_like_expr                               arms  { ($1, $2) }
 
 -- As per https://github.com/rust-lang/rust/issues/15701 (as of March 10 2017), the only way to have
 -- attributes on expressions should be with inner attributes on a paren expression.
@@ -1123,24 +1119,104 @@ field :: { Field Span }
 ----------------
 -- Statements --
 ----------------
+-- Postfix expressions that can come after an expression block, in a 'stmt'
+--
+--  * `{ 1 }[0]` isn't here because it is treated as `{ 1 }; [0]`
+--  * `{ 1 }(0)` isn't here because it is treated as `{ 1 }; (0)`
+--
+postfix_block(lhs) :: { Expr Span }
+  -- postfix expressions
+  : lhs '?'                          { Try [] $1 ($1 # $>) }
+  | lhs '.' ident       %prec FIELD  { FieldAccess [] $1 (unspan $3) ($1 # $>) }
+  | lhs '.' ident '(' sep_byT(expr,',') ')'
+    { MethodCall [] $1 (unspan $3) Nothing $5 ($1 # $>) }
+  | lhs '.' ident '::' '<' sep_byT(ty,',') '>' '(' sep_byT(expr,',') ')'
+    { MethodCall [] $1 (unspan $3) (Just $6) $9 ($1 # $>) }
+  | lhs '.' int                      {%
+      case lit $3 of
+        Int Dec i Unsuffixed _ -> pure (TupField [] $1 (fromIntegral i) ($1 # $3))
+        _ -> parseError $3
+    }
+
+gen_expression_block(lhs,rhs,rhs2) :: { Expr Span }
+  : lhs '?'                          { Try [] $1 ($1 # $>) }
+  | lhs '[' expr ']'                 { Index [] $1 $3 ($1 # $>) }
+  | lhs '(' sep_byT(expr,',') ')'    { Call [] $1 $3 ($1 # $>) }
+  | lhs '.' ident       %prec FIELD  { FieldAccess [] $1 (unspan $3) ($1 # $>) }
+  | lhs '.' ident '(' sep_byT(expr,',') ')'
+    { MethodCall [] $1 (unspan $3) Nothing $5 ($1 # $>) }
+  | lhs '.' ident '::' '<' sep_byT(ty,',') '>' '(' sep_byT(expr,',') ')'
+    { MethodCall [] $1 (unspan $3) (Just $6) $9 ($1 # $>) }
+  | lhs '.' int                      {%
+      case lit $3 of
+        Int Dec i Unsuffixed _ -> pure (TupField [] $1 (fromIntegral i) ($1 # $3))
+        _ -> parseError $3
+    }
+  -- unary expressions
+  | lhs ':' ty_no_plus               { TypeAscription [] $1 $3 ($1 # $>) }
+  | lhs as ty_no_plus                { Cast [] $1 $3 ($1 # $>) }
+  -- binary expressions
+  | lhs '*' rhs                      { Binary [] MulOp $1 $3 ($1 # $>) }
+  | lhs '/' rhs                      { Binary [] DivOp $1 $3 ($1 # $>) }
+  | lhs '%' rhs                      { Binary [] RemOp $1 $3 ($1 # $>) }
+  | lhs '+' rhs                      { Binary [] AddOp $1 $3 ($1 # $>) }
+  | lhs '-' rhs                      { Binary [] SubOp $1 $3 ($1 # $>) }
+  | lhs '<<' rhs                     { Binary [] ShlOp $1 $3 ($1 # $>) }
+  | lhs '>>' rhs                     { Binary [] ShrOp $1 $3 ($1 # $>) }
+  | lhs '&' rhs                      { Binary [] BitAndOp $1 $3 ($1 # $>) }
+  | lhs '^' rhs                      { Binary [] BitXorOp $1 $3 ($1 # $>) }
+  | lhs '|' rhs                      { Binary [] BitOrOp $1 $3 ($1 # $>) }
+  | lhs '==' rhs                     { Binary [] EqOp $1 $3 ($1 # $>) }
+  | lhs '!=' rhs                     { Binary [] NeOp $1 $3 ($1 # $>) }
+  | lhs '<'  rhs                     { Binary [] LtOp $1 $3 ($1 # $>) }
+  | lhs '>'  rhs                     { Binary [] GtOp $1 $3 ($1 # $>) }
+  | lhs '<=' rhs                     { Binary [] LeOp $1 $3 ($1 # $>) }
+  | lhs '>=' rhs                     { Binary [] GeOp $1 $3 ($1 # $>) }
+  | lhs '&&' rhs                     { Binary [] AndOp $1 $3 ($1 # $>) }
+  | lhs '||' rhs                     { Binary [] OrOp $1 $3 ($1 # $>) }
+  -- range expressions
+  | lhs '..'        %prec POSTFIXRNG { Range [] (Just $1) Nothing HalfOpen ($1 # $>) }
+  | lhs '...'       %prec POSTFIXRNG { Range [] (Just $1) Nothing Closed ($1 # $>) }
+  | lhs '..'  rhs2  %prec INFIXRNG   { Range [] (Just $1) (Just $3) HalfOpen ($1 # $>) }
+  | lhs '...' rhs2  %prec INFIXRNG   { Range [] (Just $1) (Just $3) Closed ($1 # $>) }
+  -- assignment expressions
+  | lhs '<-' rhs                     { InPlace [] $1 $3 ($1 # $>) }
+  | lhs '=' rhs                      { Assign [] $1 $3 ($1 # $>) }
+  | lhs '>>=' rhs                    { AssignOp [] ShrOp $1 $3 ($1 # $>) }
+  | lhs '<<=' rhs                    { AssignOp [] ShlOp $1 $3 ($1 # $>) }
+  | lhs '-=' rhs                     { AssignOp [] SubOp $1 $3 ($1 # $>) }
+  | lhs '+=' rhs                     { AssignOp [] AddOp $1 $3 ($1 # $>) }
+  | lhs '*=' rhs                     { AssignOp [] MulOp $1 $3 ($1 # $>) }
+  | lhs '/=' rhs                     { AssignOp [] DivOp $1 $3 ($1 # $>) }
+  | lhs '^=' rhs                     { AssignOp [] BitXorOp $1 $3 ($1 # $>) }
+  | lhs '|=' rhs                     { AssignOp [] BitOrOp $1 $3 ($1 # $>) }
+  | lhs '&=' rhs                     { AssignOp [] BitAndOp $1 $3 ($1 # $>) }
+  | lhs '%=' rhs                     { AssignOp [] RemOp $1 $3 ($1 # $>) }
+
+
+postfix_block_expr :: { Expr Span }
+  : postfix_block(block_like_expr)                         { $1 }
+  | postfix_block(vis_safety_block)                        { $1 }
+  | gen_expression_block(postfix_block_expr,expr,expr)     { $1 } 
+
+vis_safety_block :: { Expr Span }
+  : pub_or_inherited safety inner_attrs_block {%
+       let (as, Block ss r x) = $3
+           e = BlockExpr as (Block ss (unspan $2) ($2 # x)) ($2 # x)
+       in noVis $1 e
+    }
+
 
 stmt :: { Stmt Span }
   : ntStmt                                                 { $1 }
   | many(outer_attribute) let pat ':' ty initializer ';'   { Local $3 (Just $5) $6 $1 ($1 # $2 # $>) }
   | many(outer_attribute) let pat        initializer ';'   { Local $3 Nothing $4 $1 ($1 # $2 # $>) }
   | many(outer_attribute) nonblock_expr ';'                { toStmt ($1 `addAttrs` $2) True  False ($1 # $2 # $3) }
   | many(outer_attribute) block_like_expr ';'              { toStmt ($1 `addAttrs` $2) True  True  ($1 # $2 # $3) }
+  | many(outer_attribute) postfix_block_expr ';'           { toStmt ($1 `addAttrs` $2) True  True  ($1 # $2 # $3) }
   | many(outer_attribute) block_like_expr    %prec NOSEMI  { toStmt ($1 `addAttrs` $2) False True  ($1 # $2) }
-  | many(outer_attribute) pub_or_inherited safety inner_attrs_block ';' {%
-       let (as, Block ss r x) = $4 
-           e = BlockExpr ($1 ++ as) (Block ss (unspan $3) ($3 # x)) ($3 # x)
-       in noVis $2 (toStmt e True True ($1 # e # $>))
-    }
-  | many(outer_attribute) pub_or_inherited safety inner_attrs_block %prec NOSEMI {%
-       let (as, Block ss r x) = $4 
-           e = BlockExpr ($1 ++ as) (Block ss (unspan $3) ($3 # x)) ($3 # x)
-       in noVis $2 (toStmt e False True ($1 # e))
-    }
+  | many(outer_attribute) vis_safety_block ';'             { toStmt ($1 `addAttrs` $2) True True ($1 # $2 # $>) }
+  | many(outer_attribute) vis_safety_block   %prec NOSEMI  { toStmt ($1 `addAttrs` $2) False True ($1 # $2) }
   | gen_item(pub_or_inherited)                             { ItemStmt $1 (spanOf $1) }
   | many(outer_attribute) expr_path '!' ident '[' many(token_tree) ']' ';'
     { ItemStmt (macroItem $1 (Just (unspan $4)) (Mac $2 $6 ($2 # $>)) ($1 # $2 # $>)) ($1 # $2 # $>) }
@@ -1162,6 +1238,7 @@ stmts_possibly_no_semi :: { [Maybe (Stmt Span)] }
   : stmtOrSemi stmts_possibly_no_semi                      { $1 : $2 }
   | stmtOrSemi                                             { [$1] }
   | many(outer_attribute) nonblock_expr                    { [Just (toStmt ($1 `addAttrs` $2) False False ($1 # $2))] }
+  | many(outer_attribute) postfix_block_expr               { [Just (toStmt ($1 `addAttrs` $2) False True  ($1 # $2))] }
 
 initializer :: { Maybe (Expr Span) }
   : '=' expr                                               { Just $2 }
@@ -1570,6 +1647,10 @@ export_arg :: { Arg Span }
   : arg_general                                            { $1 }
   | arg_self                                               { $1 }
 
+-- Exporting 'pat' directly screws up expressions like 'x && y()'
+export_pat :: { Pat Span }
+  : pat                                                    { $1 }
+
 {
 -- | Parser for literals.
 parseLit :: P (Lit Span)
@@ -1610,9 +1691,6 @@ parseTt :: P TokenTree
 -- | Parser for lifetime definitions
 parseLifetimeDef :: P (LifetimeDef Span)
 
--- | Parser for type parameter bound
-parseTyParamBound :: P (TyParamBound Span)
-
 -- | Parser for a type parameter
 parseTyParam :: P (TyParam Span)
 

src/Language/Rust/Parser/Lexer.x

@@ -956,7 +956,7 @@ $hexit             = [0-9a-fA-F]
 @lit_float         = ( 0 @decimal_suffix | ( [1-9][0-9_]* | 0[0-9_]+ ) @decimal_suffix? ) @exponent_suffix?
 @lit_float2        = [0-9][0-9_]* \.
 
-@lit_str           = \" (\\\n | \\\r\n | \\ @char_escape | [^\"])* \"
+@lit_str           = \" (\\\n | \\\r\n | \\ @char_escape | [^\\\"] | \n | \r)* \"
 @lit_byte_str      = b @lit_str
 
 @lit_raw_str       = r \#* \"