updates

Author: chrisdickinson

lib/expr.js

@@ -100,7 +100,19 @@ module.exports = function(incoming_state, incoming_tokens) {
   if(!tokens.length) return
 
   advance()
-  expression(0)
+  var result = expression(0)
+
+  result.parent = state[0]
+  emit(result)
+
+  function emit(node) {
+    state.unshift(node, false)
+    for(var i = 0, len = node.children.length; i < len; ++i) {
+      emit(node.children[i])
+    }
+    state.shift()
+  }
+
 }
 
 function symbol(id, binding_power) {

lib/index.js

@@ -14,6 +14,8 @@ function create_stream() {
 // singleton!
 var Advance = new Object
 
+var DEBUG = false
+
 var _ = 0
   , STMT = _++
   , STMTLIST = _++
@@ -45,9 +47,9 @@ var NO_ASSIGN_ALLOWED = false
 
 // map of tokens to stmt types
 var token_map = {
-  'block-comment': COMMENT
-, 'line-comment': COMMENT
-, 'preprocessor': PREPROCESSOR
+    'block-comment': COMMENT
+  , 'line-comment': COMMENT
+  , 'preprocessor': PREPROCESSOR
 }
 
 // map of stmt types to human
@@ -137,15 +139,16 @@ function parser() {
   // stream functions ---------------------------------------------
 
   function write(input) {
-    if(input.type === 'whitespace') {
+    if(input.type === 'whitespace' || input.type === 'line-comment' || input.type === 'block-comment') {
       whitespace.push(input)
       return
     }
     tokens.push(input)
     token = token || tokens[0]
 
-    if(node) {
-      node.whitespace.concat(whitespace.splice(0, whitespace.length))
+    if(token && whitespace.length) {
+      token.preceding = token.preceding || []
+      token.preceding.concat(whitespace.splice(0, whitespace.length))
     }
 
     while(take()) switch(state[0].mode) {
@@ -205,10 +208,22 @@ function parser() {
     state.shift()
   }
 
-  function special_unshift(_node) {
+  function special_unshift(_node, add_child) {
+    _node.parent = state[0]
+
     var ret = [].unshift.call(this, _node)
 
-    node.children.push(_node)
+    add_child = add_child === undefined ? true : add_child
+
+    if(DEBUG) {
+      var pad = ''
+      for(var i = 0, len = this.length - 1; i < len; ++i) {
+        pad += ' |'
+      }
+      console.log(pad, '\\'+_node.type)
+    }
+
+    if(add_child) node.children.push(_node)
     node = _node
 
     return ret
@@ -219,6 +234,14 @@ function parser() {
       , okay = check[this.length]
       , emit = false
 
+    if(DEBUG) {
+      var pad = ''
+      for(var i = 0, len = this.length; i < len; ++i) {
+        pad += ' |'
+      }
+      console.log(pad, '/'+_node.type)
+    }
+
     if(check.length) { 
       if(okay !== undefined) {
         emit = okay.test ? okay.test(_node.type) : okay === _node.type
@@ -227,7 +250,7 @@ function parser() {
       emit = true
     }
 
-    if(emit) stream.emit('data', node) 
+    if(emit) stream.emit('data', _node) 
 
     node = _node.parent
     return _node
@@ -248,8 +271,6 @@ function parser() {
         return state.scope.exit(), state.shift()
       }
       switch(token.type) {
-        case 'block-comment':
-        case 'line-comment':
         case 'preprocessor':
           state.fake(adhoc())
           tokens.shift()
@@ -329,10 +350,6 @@ function parser() {
       state[0].got = []
     }
 
-    if(token.type === 'block-comment' || token.type === 'line-comment') {
-      return tokens.shift()
-    }
-
     state[0].got.push(tokens.shift())
 
     if(state[0].need !== state[0].got.length)
@@ -344,8 +361,20 @@ function parser() {
         token = state[0].got.shift(), state.fake(keyword())
       } else {
         state.fake(keyword('<default>'))
-      }   
-      token = state[0].got.shift(), state.fake(keyword())
+      }
+      token = state[0].got.shift()
+
+      if(token.type === 'keyword') { 
+        state.fake(keyword())
+      } else {
+        var lookup = state.scope.find(token.data)
+
+        if(!lookup) return
+
+        lookup = Object.create(lookup)
+        lookup.token = token
+        state.fake(lookup)
+      }
 
       while(state[0].got.length)
         tokens.unshift(token = state[0].got.pop())
@@ -354,7 +383,19 @@ function parser() {
     }
 
     // it's a function
-    token = state[0].got.shift(), state.fake(keyword())
+    token = state[0].got.shift()
+      
+    if(token.type === 'keyword') {
+      state.fake(keyword())
+    } else {
+      var lookup = state.scope.find(token.data)
+
+      if(!lookup) return
+
+      lookup = Object.create(lookup)
+      lookup.token = token
+      state.fake(lookup)
+    }
     // replace the last two tokens
     tokens.unshift(state[0].got.pop())
     tokens.unshift(token = state[0].got.pop())
@@ -364,10 +405,6 @@ function parser() {
 
   function parse_decllist() {
     // grab ident
-    if(token.type === 'line-comment' || token.type === 'block-comment') {
-      return tokens.shift()
-    }
-
     if(token.type === 'ident') {
       var name = token.data
       state.unshift(ident())
@@ -432,10 +469,6 @@ function parser() {
   function parse_expr() {
     var expecting = state[0].expecting
 
-    if(token.type === 'block-comment' || token.type === 'line-comment')
-      return tokens.shift()
-
-
     state[0].tokens = state[0].tokens || []
 
     if(state[0].parenlevel === undefined) {
@@ -520,7 +553,7 @@ function parser() {
   function assert_null_string_or_array(x, y) {
     switch(typeof x) {
       case 'string': if(y !== x) {
-        unexpected('expected `'+x+'`, got '+y);
+        unexpected('expected `'+x+'`, got '+y+'\n'+token.data);
       } return !errored
 
       case 'object': if(x && x.indexOf(y) === -1) {
@@ -540,10 +573,6 @@ function parser() {
     return function() {
       var current = state[0]
 
-      if(token.type === 'line-comment' || token.type === 'block-comment') {
-        return tokens.shift()
-      }
-
       current.stage || (current.stage = 0)
 
       step = steps[current.stage]
@@ -570,7 +599,12 @@ function parser() {
       var name = token.data
       return assert('ident') && (state.unshift(ident()), state.scope.define(name), Advance)
     } :  function() {
-      return assert('ident') && (state.fake(state.scope.find(token.data)), tokens.shift(), Advance)
+      if(!assert('ident')) return
+
+      var s = Object.create(state.scope.find(token.data))
+      s.token = token
+
+      return (tokens.shift(), Advance)
     }
   }
 
@@ -744,7 +778,6 @@ function mknode(mode, sourcetoken, parent) {
     , token: sourcetoken
     , parent: parent
     , children: []
-    , whitespace: []
     , type: stmt_type[mode]
     , id: (Math.random() * 0xFFFFFFFF).toString(16)
   }

test/index.js

@@ -4,9 +4,20 @@ var tokenizer = require('glsl-tokenizer')()
   , path = require('path').join(__dirname, 'test.glsl')
 
 var num = 0
+  , types = []
 
 fs.createReadStream(path)
   .pipe(tokenizer)
   .pipe(parser())
   .on('data', function(x) {
+    console.log(selector(x))
   })
+
+function selector(x) {
+  var list = []
+  do {
+    list.unshift(x.type)
+  } while(x = x.parent)
+
+  return list.join('/')
+}