Bump to Rust 1.19

Update the AST to the one in rust nightly 1.19. Notable changes are:

  * 'TokenStream' is a new type (core to Macros 2.0)
  * 'MetaItem' and 'NestedMetaItem' are gone
  * 'Attribute' now just contains a path and a 'TokenStream'
  * Some lexer peculiarities around literals are fixed

Author: harpocrates

benchmarks/allocation-benchmarks/Main.hs

@@ -19,8 +19,7 @@ import System.IO.Error (isDoesNotExistError)
 import Data.Aeson
 import qualified Data.ByteString.Lazy as BL
 
--- TODO:
--- Only allocation and GCs seem to be really reproducible. Live and max sometimes are 0.
+-- TODO: only allocation and GCs seem to be really reproducible. Live and max sometimes are 0.
 
 main :: IO ()
 main = do

language-rust.cabal

@@ -48,6 +48,7 @@ library
                        Language.Rust.Parser.Literals
                        Language.Rust.Parser.Reversed
                        Language.Rust.Pretty.Resolve
+                       Language.Rust.Pretty.Literals
                        Language.Rust.Syntax.AST
                        Language.Rust.Syntax.Ident
                        Language.Rust.Syntax.Token

sample-sources/attributes.rs

@@ -91,8 +91,8 @@ fn expressions() {
   #[blockexpr_outer]
   unsafe { #![blockexpr_inner] 1; 2 }
 
-//  #[catch_outer]  UPDATE RUSTC
-//  do catch { #![catch_inner] 1 }
+  #[catch_outer]
+  do catch { #![catch_inner] 1 };
 
 //  #[assign_outer]
 //  x = 1;

sample-sources/expressions.rs

@@ -41,6 +41,7 @@ fn main() {
   let x = [1; 5];
   let x = 1 * (2 + 3);
   let x = foo()?;
+  let x = do catch { 1 };
   return 0;
   return;
 

sample-sources/macros.rs

@@ -21,22 +21,28 @@ tokentrees!{
 
 // literals
 literals!{
-  b'a' b'\n' b'a'suffix
-  'a' '\n' 'a'suffix
-  
-  123 123i32
-  0b1100_1101 0b1100_1101isize
-  0o3170 0o3170i64
-  0xAFAC 0xAFACu32
-  
-  123.1 123.1f32 
-  123.f32          // Gets parsed as [integer token "123", ".", ident token "f32"]
-  123.0f32
-  0e+10            // Gets parsed as [integer token "0" with suffix "e", "+", integer token "10"]
-  00e+10
-  9e+10
-  123e-9f32
- 
+  characters_bytes!{
+    b'a' b'\n' b'a'suffix
+    'a' '\n' 'a'suffix
+  }
+
+  integral_numbers!{
+    123 123i32
+    0b1100_1101 0b1100_1101isize
+    0o3170 0o3170i64
+    0xAFAC 0xAFACu32
+  }
+
+  float_numbers!{
+    123.1 123.1f32 
+    123.f32          // Gets parsed as [integer token "123", ".", ident token "f32"]
+    123.0f32
+    0e+10
+    00e+10
+    9e+10
+    123e-9f32
+  }
+
   strings!{
     "hello \n world!"
     
@@ -77,11 +83,11 @@ fn main() {
   print!("{}\n", 001e+1);
   print!("{}\n", 123.);
   print!("{}\n", 123.0f32);
+  print!("{}\n", 0e+1);
   print!("{}\n", 123.0__9);
   print!("{}\n", 123e-4f32);
 
   // Not what they look like, see above
-  print!("{}\n", 0e+1);
   print!("{}\n", 123.f32);
 }
 

src/Language/Rust/Parser.hs

@@ -82,6 +82,7 @@ instance Parse (Stmt Span) where parser = parseStmt
 instance Parse (Item Span) where parser = parseItem
 instance Parse (SourceFile Span) where parser = parseSourceFile
 instance Parse TokenTree where parser = parseTt
+instance Parse TokenStream where parser = parseTokenStream
 instance Parse (Block Span) where parser = parseBlock
 instance Parse (ImplItem Span) where parser = parseImplItem 
 instance Parse (TraitItem Span) where parser = parseTraitItem