Refactor prelexer and improve performance

Implements some functions we previously used from
the standard implementation to be locale unaware.
This also improved the performance by quite a bit.
Also fixes word boundary to let `-` be part of words

Author: mgreter

Makefile

@@ -122,6 +122,7 @@ SOURCES = \
 	file.cpp \
 	functions.cpp \
 	inspect.cpp \
+	lexer.cpp \
 	node.cpp \
 	json.cpp \
 	emitter.cpp \

Makefile.am

@@ -65,6 +65,7 @@ libsass_la_SOURCES = \
 	file.cpp file.hpp \
 	functions.cpp functions.hpp \
 	inspect.cpp inspect.hpp \
+	lexer.cpp lexer.hpp \
 	node.cpp node.hpp \
 	json.cpp json.hpp \
 	emitter.cpp emitter.hpp \

ast.hpp

@@ -460,14 +460,14 @@ namespace Sass {
   class Assignment : public Statement {
     ADD_PROPERTY(string, variable);
     ADD_PROPERTY(Expression*, value);
-    ADD_PROPERTY(bool, is_guarded);
+    ADD_PROPERTY(bool, is_default);
     ADD_PROPERTY(bool, is_global);
   public:
     Assignment(ParserState pstate,
                string var, Expression* val,
-               bool guarded = false,
-               bool global = false)
-    : Statement(pstate), variable_(var), value_(val), is_guarded_(guarded), is_global_(global)
+               bool is_default = false,
+               bool is_global = false)
+    : Statement(pstate), variable_(var), value_(val), is_default_(is_default), is_global_(is_global)
     { }
     ATTACH_OPERATIONS();
   };

bind.cpp

@@ -20,7 +20,7 @@ namespace Sass {
       Parameter*  p = (*ps)[i];
       param_map[p->name()] = p;
       // if (p->default_value()) {
-      //   env->current_frame()[p->name()] = p->default_value()->perform(eval->with(env));
+      //   env->local_frame()[p->name()] = p->default_value()->perform(eval->with(env));
       // }
     }
 
@@ -41,12 +41,12 @@ namespace Sass {
       if (p->is_rest_parameter()) {
         if (a->is_rest_argument()) {
           // rest param and rest arg -- just add one to the other
-          if (env->current_frame_has(p->name())) {
-            *static_cast<List*>(env->current_frame()[p->name()])
+          if (env->has_local(p->name())) {
+            *static_cast<List*>(env->local_frame()[p->name()])
             += static_cast<List*>(a->value());
           }
           else {
-            env->current_frame()[p->name()] = a->value();
+            env->local_frame()[p->name()] = a->value();
           }
         } else {
 
@@ -55,7 +55,7 @@ namespace Sass {
                                              0,
                                              List::COMMA,
                                              true);
-          env->current_frame()[p->name()] = arglist;
+          env->local_frame()[p->name()] = arglist;
           while (ia < LA) {
             a = (*as)[ia];
             (*arglist) << new (ctx.mem) Argument(a->pstate(),
@@ -99,7 +99,7 @@ namespace Sass {
             msg << callee << " has no parameter named " << name;
             error(msg.str(), a->pstate());
           }
-          env->current_frame()[name] = argmap->at(key);
+          env->local_frame()[name] = argmap->at(key);
         }
         ++ia;
         continue;
@@ -108,14 +108,14 @@ namespace Sass {
       }
 
       if (a->name().empty()) {
-        if (env->current_frame_has(p->name())) {
+        if (env->has_local(p->name())) {
           stringstream msg;
           msg << "parameter " << p->name()
           << " provided more than once in call to " << callee;
           error(msg.str(), a->pstate());
         }
         // ordinal arg -- bind it to the next param
-        env->current_frame()[p->name()] = a->value();
+        env->local_frame()[p->name()] = a->value();
         ++ip;
       }
       else {
@@ -131,13 +131,13 @@ namespace Sass {
               << "cannot be used as named argument";
           error(msg.str(), a->pstate());
         }
-        if (env->current_frame_has(a->name())) {
+        if (env->has_local(a->name())) {
           stringstream msg;
           msg << "parameter " << p->name()
               << "provided more than once in call to " << callee;
           error(msg.str(), a->pstate());
         }
-        env->current_frame()[a->name()] = a->value();
+        env->local_frame()[a->name()] = a->value();
       }
     }
 
@@ -150,9 +150,9 @@ namespace Sass {
       // cerr << "env for default params:" << endl;
       // env->print();
       // cerr << "********" << endl;
-      if (!env->current_frame_has(leftover->name())) {
+      if (!env->has_local(leftover->name())) {
         if (leftover->is_rest_parameter()) {
-          env->current_frame()[leftover->name()] = new (ctx.mem) List(leftover->pstate(),
+          env->local_frame()[leftover->name()] = new (ctx.mem) List(leftover->pstate(),
                                                                       0,
                                                                       List::COMMA,
                                                                       true);
@@ -167,7 +167,7 @@ namespace Sass {
           eval->backtrace = old_bt;
           eval->contextualize = old_context;
           // dv->perform(&to_string);
-          env->current_frame()[leftover->name()] = dv;
+          env->local_frame()[leftover->name()] = dv;
         }
         else {
           // param is unbound and has no default value -- error

environment.hpp

@@ -18,83 +18,148 @@ namespace Sass {
   template <typename T>
   class Environment {
     // TODO: test with unordered_map
-    map<string, T> current_frame_;
+    map<string, T> local_frame_;
     ADD_PROPERTY(Environment*, parent);
 
   public:
     Memory_Manager<AST_Node> mem;
-    Environment() : current_frame_(map<string, T>()), parent_(0) { }
-
-    map<string, T>& current_frame() { return current_frame_; }
+    Environment() : local_frame_(map<string, T>()), parent_(0) { }
 
+    // link parent to create a stack
     void link(Environment& env) { parent_ = &env; }
     void link(Environment* env) { parent_ = env; }
 
-    bool has(const string key) const
+    // this is used to find the global frame
+    // which is the second last on the stack
+    bool is_lexical() const
     {
-      if (current_frame_.count(key))  return true;
-      else if (parent_)               return parent_->has(key);
-      else                            return false;
+      return !! parent_ && parent_->parent_;
     }
 
-    bool current_frame_has(const string key) const
-    { return !!current_frame_.count(key); }
-
-    void current_frame_set(const string key, T val)
-    { current_frame_[key] = val; }
+    // only match the real root scope
+    // there is still a parent around
+    // not sure what it is actually use for
+    // I guess we store functions etc. there
+    bool is_root_scope() const
+    {
+      return parent_ && ! parent_->parent_;
+    }
 
-    void global_frame_set(const string key, T val)
-    { global_frame()->current_frame_[key] = val; }
+    // scope operates on the current frame
 
-    Environment* grandparent() const
-    {
-      if(parent_ && parent_->parent_) return parent_->parent_;
-      else return 0;
+    map<string, T>& local_frame() {
+      return local_frame_;
     }
 
-    Environment* global_frame()
+    bool has_local(const string& key) const
+    { return local_frame_.count(key); }
+
+    T& get_local(const string& key)
+    { return local_frame_[key]; }
+
+    void set_local(const string& key, T val)
+    { local_frame_[key] = val; }
+
+    void del_local(const string& key)
+    { local_frame_.erase(key); }
+
+
+    // global operates on the global frame
+    // which is the second last on the stack
+
+    Environment* global_env()
     {
       Environment* cur = this;
-      // looks like global variables
-      // are in the second last parent
-      while (cur->grandparent()) {
+      while (cur->is_lexical()) {
         cur = cur->parent_;
       }
       return cur;
     }
 
-    bool global_frame_has(const string key) const
+    bool has_global(const string& key)
+    { return global_env()->has(key); }
+
+    T& get_global(const string& key)
+    { return (*global_env())[key]; }
+
+    void set_global(const string& key, T val)
+    { global_env()->local_frame_[key] = val; }
+
+    void del_global(const string& key)
+    { global_env()->local_frame_.erase(key); }
+
+    // see if we have a lexical variable
+    // move down the stack but stop before we
+    // reach the global frame (is not included)
+    bool has_lexical(const string& key) const
+    {
+      auto cur = this;
+      while (cur->is_lexical()) {
+        if (cur->has_local(key)) return true;
+        cur = cur->parent_;
+      }
+      return false;
+    }
+
+    // see if we have a lexical we could update
+    // either update already existing lexical value
+    // or if flag is set, we create one if no lexical found
+    void set_lexical(const string& key, T val, bool global = false)
     {
-      if(parent_ && !grandparent()) {
-        return has(key);
+      auto cur = this;
+      while (cur->is_lexical()) {
+        if (cur->has_local(key)) {
+          cur->set_local(key, val);
+          return;
+        }
+        cur = cur->parent_;
       }
-      else if(parent_) {
-        return parent_->global_frame_has(key);
+      if (global) {
+        set_global(key, val);
+      } else {
+        set_local(key, val);
       }
-      else {
-        return false;
+    }
+
+    // look on the full stack for key
+    // include all scopes available
+    bool has(const string& key) const
+    {
+      auto cur = this;
+      while (cur) {
+        if (cur->has_local(key)) {
+          return true;
+        }
+        cur = cur->parent_;
       }
+      return false;
     }
 
-    T& operator[](const string key)
+    // use array access for getter and setter functions
+    T& operator[](const string& key)
     {
-      if (current_frame_.count(key))  return current_frame_[key];
-      else if (parent_)               return (*parent_)[key];
-      else                            return current_frame_[key];
+      auto cur = this;
+      while (cur) {
+        if (cur->has_local(key)) {
+          return cur->get_local(key);
+        }
+        cur = cur->parent_;
+      }
+      return get_local(key);
     }
 
-#ifdef DEBUG
+    #ifdef DEBUG
     void print()
     {
-      for (typename map<string, T>::iterator i = current_frame_.begin(); i != current_frame_.end(); ++i) {
+      for (typename map<string, T>::iterator i = local_frame_.begin(); i != local_frame_.end(); ++i) {
         cerr << i->first << endl;
       }
       if (parent_) {
         cerr << "---" << endl;
         parent_->print();
       }
     }
-#endif
+    #endif
 
   };
 }