Add major re-factoring lexer in parser.hpp

More modular and hopefully better readable!

Author: mgreter

parser.hpp

@@ -72,42 +72,41 @@ namespace Sass {
     bool peek_newline(const char* start = 0);
 
     template <prelexer mx>
-    const char* peek(const char* start = 0)
+    const char* sneak(const char* start = 0)
     {
-      if (!start) start = position;
-      const char* it_before_token;
-      if (mx == block_comment) {
-        it_before_token = // start;
-          zero_plus< alternatives<spaces, line_comment> >(start);
-      }
-      else if (/*mx == ancestor_of ||*/ mx == no_spaces) {
-        it_before_token = position;
-      }
-      else if (mx == spaces) {
-        it_before_token = mx(start);
-        if (it_before_token) {
-          return it_before_token;
-        }
-        else {
-          return 0;
-        }
-      }
-      else if (mx == optional_spaces) {
-        it_before_token = optional_spaces(start);
-      }
-      else if (mx == line_comment_prefix || mx == block_comment_prefix) {
-        it_before_token = position;
-      }
-      else {
-        it_before_token = optional_css_whitespace(start);
-      }
-      const char* it_after_token = mx(it_before_token);
-      if (it_after_token) {
-        return it_after_token;
-      }
-      else {
-        return 0;
+
+      // maybe use optional start position from arguments?
+      const char* it_before_token = start ? start : position;
+
+      // skip white-space?
+      if (mx == url ||
+          mx == spaces ||
+          mx == no_spaces ||
+          mx == css_comments ||
+          mx == css_whitespace ||
+          mx == optional_spaces ||
+          mx == optional_css_comments ||
+          mx == optional_css_whitespace
+      ) {
+        return it_before_token;
       }
+
+      // skip over spaces, tabs and sass line comments
+      return optional_css_whitespace(it_before_token);
+
+    }
+
+    template <prelexer mx>
+    const char* peek(const char* start = 0)
+    {
+
+      // sneak up to the actual token we want to lex
+      // this should skip over white-space if desired
+      const char* it_before_token = sneak < mx >(start);
+
+      // match the given prelexer
+      return mx(it_before_token);
+
     }
 
     // white-space handling is built into the lexer
@@ -120,92 +119,51 @@ namespace Sass {
       // remeber interesting position
       const char* wspace_start = position;
 
-      // advance position for next call
-      before_token = after_token;
-
-      // after optional whitespace
-      const char* it_before_token;
-
-      if (mx == block_comment) {
-        // a block comment can be preceded by spaces and/or line comments
-        it_before_token = zero_plus< alternatives<spaces, line_comment> >(position);
-      }
-      else if (mx == url || mx == no_spaces) {
-        // parse everything literally
-        it_before_token = position;
-      }
-
-      else if (mx == spaces) {
-        it_before_token = spaces(position);
-        if (it_before_token) {
-          return position = it_before_token;
-        }
-        else {
-          return 0;
-        }
-      }
-
-      else if (mx == css_comments ||
-               mx == css_whitespace ||
-               mx == optional_css_comments ||
-               mx == optional_css_whitespace
-      ) {
-        it_before_token = position;
-      }
+      // sneak up to the actual token we want to lex
+      // this should skip over white-space if desired
+      const char* it_before_token = sneak < mx >(position);
 
-      else if (mx == optional_spaces) {
-        // ToDo: what are optiona_spaces ???
-        it_before_token = optional_spaces(position);
-      }
-      else {
-        // most can be preceded by spaces and comments
-        it_before_token = optional_css_whitespace(position);
-      }
+      // advance position (add whitespace before current token)
+      before_token = after_token.inc(position, it_before_token);
 
       // now call matcher to get position after token
       const char* it_after_token = mx(it_before_token);
+
       // assertion that we got a valid match
       if (it_after_token == 0) return 0;
 
-      // add whitespace after previous and before this token
-      while (position < it_before_token && *position) {
-        if (*position == '\n') {
-          ++ before_token.line;
-          before_token.column = 0;
-        } else {
-          ++ before_token.column;
-        }
-        ++position;
-      }
-
-      // copy position
-      after_token = before_token;
-
-      Offset size(0, 0);
-
-      // increase position to include current token
-      while (position < it_after_token && *position) {
-        if (*position == '\n') {
-          ++ size.line;
-          size.column = 0;
-        } else {
-          ++ size.column;
-        }
-        ++position;
-      }
+      // create new lexed token object (holds all parse result information)
+      lexed = Token(wspace_start, it_before_token, it_after_token);
 
-      after_token = after_token + size;
+      // update position of after_token (add token to before position)
+      after_token = before_token.inc(it_before_token, it_after_token);
 
-      // create parsed token string (public member)
-      lexed = Token(wspace_start, it_before_token, it_after_token);
-      Position pos(before_token.file, before_token.line, before_token.column);
-      pstate = ParserState(path, lexed, pos, size);
+      // ToDo: could probably do this incremetal on original object
+      pstate = ParserState(path, lexed, before_token, after_token - before_token);
 
       // advance internal char iterator
       return position = it_after_token;
 
     }
 
+    // skips over css comments
+    template <prelexer mx>
+    const char* lex_css()
+    {
+      // throw away comments
+      lex < css_comments >();
+      // now lex a token
+      return lex< mx >();
+    }
+
+    // skips over css comments
+    template <prelexer mx>
+    const char* peek_css()
+    {
+      // now peek a token (skip comments first)
+      return peek< mx >(peek < css_comments >());
+    }
+
 #ifdef __clang__
 
 #pragma clang diagnostic pop