Merge pull request #2541 from mgreter/bugfix/sourcemap/unicode

Make sourcemaps fully unicode aware

Author: mgreter

src/emitter.cpp

@@ -103,7 +103,11 @@ namespace Sass {
   // prepend some text or token to the buffer
   void Emitter::prepend_string(const std::string& text)
   {
-    wbuf.smap.prepend(Offset(text));
+    // do not adjust mappings for utf8 bom
+    // seems they are not counted in any UA
+    if (text.compare("\xEF\xBB\xBF") != 0) {
+      wbuf.smap.prepend(Offset(text));
+    }
     wbuf.buffer = text + wbuf.buffer;
   }
 

src/position.cpp

@@ -37,7 +37,6 @@ namespace Sass {
 
   // increase offset by given string (mostly called by lexer)
   // increase line counter and count columns on the last line
-  // ToDo: make the col count utf8 aware
   Offset Offset::add(const char* begin, const char* end)
   {
     if (end == 0) return *this;
@@ -47,9 +46,23 @@ namespace Sass {
         // start new line
         column = 0;
       } else {
-        ++ column;
+        // do not count any utf8 continuation bytes
+        // https://stackoverflow.com/a/9356203/1550314
+        // https://en.wikipedia.org/wiki/UTF-8#Description
+        unsigned char chr = *begin;
+        // skip over 10xxxxxx
+        // is 1st bit not set
+        if ((chr & 128) == 0) {
+          // regular ascii char
+          column += 1;
+        }
+        // is 2nd bit not set
+        else if ((chr & 64) == 0) {
+          // first utf8 byte
+          column += 1;
+        }
       }
-      ++begin;
+      ++ begin;
     }
     return *this;
   }

src/source_map.cpp

@@ -136,7 +136,7 @@ namespace Sass {
         }
       }
     }
-    // will adjust the offset
+    // adjust the buffer offset
     prepend(Offset(out.buffer));
     // now add the new mappings
     VECTOR_UNSHIFT(mappings, out.smap.mappings);