Merge pull request #1464 from mgreter/feature/unicode-sequence-parser

Add unicode sequence parsing

Author: xzyfer

src/context.cpp

@@ -32,7 +32,6 @@
 #include "sass2scss.h"
 #include "prelexer.hpp"
 #include "emitter.hpp"
-#include "debugger.hpp"
 
 namespace Sass {
   using namespace Constants;

src/prelexer.cpp

@@ -70,14 +70,23 @@ namespace Sass {
     {
       return sequence<
         exactly<'\\'>,
-        any_char
+        alternatives <
+          minmax_range<
+            3, 3, xdigit
+          >,
+          any_char
+        >,
+        optional <
+          exactly <' '>
+        >
       >(src);
     }
 
     // Match identifier start
     const char* identifier_alpha(const char* src)
     {
       return alternatives<
+               unicode_seq,
                alpha,
                unicode,
                exactly<'-'>,
@@ -90,6 +99,7 @@ namespace Sass {
     const char* identifier_alnum(const char* src)
     {
       return alternatives<
+               unicode_seq,
                alnum,
                unicode,
                exactly<'-'>,
@@ -173,6 +183,7 @@ namespace Sass {
               re_linebreak
             >,
             escape_seq,
+            unicode_seq,
             // skip interpolants
             interpolant,
             // skip non delimiters
@@ -196,6 +207,7 @@ namespace Sass {
               re_linebreak
             >,
             escape_seq,
+            unicode_seq,
             // skip interpolants
             interpolant,
             // skip non delimiters
@@ -880,6 +892,20 @@ namespace Sass {
       return (p == 0) ? t.end : 0;
     }
 
+    const char* unicode_seq(const char* src) {
+      return sequence <
+        alternatives <
+          exactly< 'U' >,
+          exactly< 'u' >
+        >,
+        exactly< '+' >,
+        padded_token <
+          6, xdigit,
+          exactly < '?' >
+        >
+      >(src);
+    }
+
     const char* static_component(const char* src) {
       return alternatives< identifier,
                            static_string,
@@ -974,5 +1000,35 @@ namespace Sass {
       return sequence< number, optional_spaces, exactly<'/'>, optional_spaces, number >(src);
     }
 
+    template <size_t size, prelexer mx, prelexer pad>
+    const char* padded_token(const char* src)
+    {
+      size_t got = 0;
+      const char* pos = src;
+      while (got < size) {
+        if (!mx(pos)) break;
+        ++ pos; ++ got;
+      }
+      while (got < size) {
+        if (!pad(pos)) break;
+        ++ pos; ++ got;
+      }
+      return got ? pos : 0;
+    }
+
+    template <size_t min, size_t max, prelexer mx>
+    const char* minmax_range(const char* src)
+    {
+      size_t got = 0;
+      const char* pos = src;
+      while (got < max) {
+        if (!mx(pos)) break;
+        ++ pos; ++ got;
+      }
+      if (got < min) return 0;
+      if (got > min) return 0;
+      return pos;
+    }
+
   }
 }

src/prelexer.hpp

@@ -227,6 +227,8 @@ namespace Sass {
     const char* kwd_charset_directive(const char* src);
     const char* kwd_extend(const char* src);
 
+    const char* unicode_seq(const char* src);
+
     const char* kwd_if_directive(const char* src);
     const char* kwd_else_directive(const char* src);
     const char* elseif_directive(const char* src);
@@ -388,6 +390,12 @@ namespace Sass {
       return counter;
     }
 
+    template <size_t size, prelexer mx, prelexer pad>
+    const char* padded_token(const char* src);
+
+    template <size_t min, size_t max, prelexer mx>
+    const char* minmax_range(const char* src);
+
   }
 }
 

src/util.cpp

@@ -363,6 +363,8 @@ namespace Sass {
           // ToDo: Maybe we could do this without creating a substring
           uint32_t cp = strtol(s.substr (i + 1, len - 1).c_str(), nullptr, 16);
 
+          if (s[i + len] == ' ') ++ len;
+
           // assert invalid code points
           if (cp == 0) cp = 0xFFFD;
           // replace bell character