Fixes #116:

- Ensure that afterNodes matchOp will realize if certain slots are actually valid or not.
  If the slots are not valid, then don't run the actual matchOp, which is what the non
  after's case is supposed to be
- Also fixed a bug where after's matchOp wasn't pulling out an object fastval correctly
  as the implementation was not there
- Fix matcher Reset() where it wipes out old slots
- Added new test data

Author: nelio2k

fastMatcher.go

@@ -11,6 +11,8 @@ type slotData struct {
 	size  int
 }
 
+var emptySlotData slotData
+
 type FastMatcher struct {
 	def     MatchDef
 	slots   []slotData
@@ -27,7 +29,9 @@ func NewFastMatcher(def *MatchDef) *FastMatcher {
 }
 
 func (m *FastMatcher) Reset() {
-	m.slots = m.slots[:0]
+	for i := 0; i < m.def.NumSlots; i++ {
+		m.slots[i] = emptySlotData
+	}
 	m.buckets.Reset()
 }
 
@@ -98,6 +102,12 @@ func (m *FastMatcher) literalFromSlot(slot SlotID) FastVal {
 	if isLiteralToken(token) {
 		var parser fastLitParser
 		value = parser.Parse(token, tokenData)
+	} else if slotInfo.size > 0 {
+		if token == tknObjectStart {
+			value = NewObjectFastVal(m.tokens.data[slotInfo.start : slotInfo.start+slotInfo.size])
+		} else if token == tknArrayStart {
+			value = NewArrayFastVal(m.tokens.data[slotInfo.start : slotInfo.start+slotInfo.size])
+		}
 	}
 
 	m.tokens.Seek(savePos)
@@ -222,20 +232,38 @@ func (m *FastMatcher) matchOp(op *OpNode, litVal *FastVal) error {
 		return nil
 	}
 
+	var slotNotFound bool
 	lhsVal := NewMissingFastVal()
 	if op.Lhs != nil {
 		lhsVal = m.resolveParam(op.Lhs, litVal)
+		if _, ok := op.Lhs.(SlotRef); ok && lhsVal.IsMissing() {
+			slotNotFound = true
+		}
 	} else if litVal != nil {
 		lhsVal = *litVal
 	}
 
 	rhsVal := NewMissingFastVal()
 	if op.Rhs != nil {
 		rhsVal = m.resolveParam(op.Rhs, litVal)
+		if _, ok := op.Rhs.(SlotRef); ok && rhsVal.IsMissing() {
+			slotNotFound = true
+		}
 	} else if litVal != nil {
 		rhsVal = *litVal
 	}
 
+	if slotNotFound {
+		// If references are for slots and at least one wasn't found
+		// then the matchOp should not execute
+		m.buckets.MarkNode(bucketIdx, false)
+
+		if m.buckets.IsResolved(0) {
+			return nil
+		}
+		return nil
+	}
+
 	var opRes bool
 	switch op.Op {
 	case OpTypeEquals:
@@ -548,7 +576,7 @@ func (m *FastMatcher) matchExec(token tokenType, tokenData []byte, tokenDataLen
 			}
 		}
 	} else if token == tknObjectStart {
-		objStartPos := m.tokens.Position()
+		objStartPos := m.tokens.Position() - 1 /* to include the objStart itself*/
 		if len(node.Elems) == 0 {
 			// If we have no element handlers, we can just skip the whole thing...
 			m.skipValue(token)
@@ -569,6 +597,7 @@ func (m *FastMatcher) matchExec(token tokenType, tokenData []byte, tokenDataLen
 		objEndPos := m.tokens.Position()
 
 		objFastVal := NewObjectFastVal(m.tokens.data[objStartPos:objEndPos])
+
 		for _, op := range node.Ops {
 			err := m.matchOp(&op, &objFastVal)
 			if err != nil {
@@ -580,7 +609,7 @@ func (m *FastMatcher) matchExec(token tokenType, tokenData []byte, tokenDataLen
 			}
 		}
 	} else if token == tknArrayStart {
-		arrayStartPos := m.tokens.Position()
+		arrayStartPos := m.tokens.Position() - 1 // Should be -1 to include the [
 		if len(node.Loops) == 0 {
 			err, shouldReturn := m.matchObjectOrArray(token, tokenData, node)
 			if shouldReturn {
@@ -699,6 +728,7 @@ func (m *FastMatcher) matchObjectOrArray(token tokenType, tokenData []byte, node
 		if err != nil {
 			return err, true
 		}
+
 		// Keep this here to catch any empty array or empty objs
 		if token == endToken {
 			return nil, true

fastval.go

@@ -438,6 +438,7 @@ func (val FastVal) compareObjArrData(other FastVal) int {
 	}
 }
 
+// Returns compared val and boolean indicating if the comparison is valid
 func (val FastVal) Compare(other FastVal) int {
 	switch val.dataType {
 	case IntValue:
@@ -480,7 +481,6 @@ func (val FastVal) Compare(other FastVal) int {
 }
 
 func (val FastVal) Equals(other FastVal) bool {
-	// TODO: I doubt this logic is correct...
 	return val.Compare(other) == 0
 }
 

filterExprParser_test.go

@@ -4,7 +4,10 @@ package gojsonsm
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
 	"testing"
 )
 
@@ -877,3 +880,52 @@ func TestFilterExpressionParser(t *testing.T) {
 	_, err = fe.OutputExpression()
 	assert.NotNil(err)
 }
+
+func readJsonHelper(fileName string) (retMap map[string]interface{}, byteSlice []byte, err error) {
+	byteSlice, err = ioutil.ReadFile(fileName)
+	if err != nil {
+		return
+	}
+	var unmarshalledIface interface{}
+	err = json.Unmarshal(byteSlice, &unmarshalledIface)
+	if err != nil {
+		return
+	}
+
+	retMap = unmarshalledIface.(map[string]interface{})
+	return
+}
+
+func TestEdgyJson(t *testing.T) {
+	assert := assert.New(t)
+	dirPath := "testdata/edgyJson"
+	edgyJsonDir, err := os.Open(dirPath)
+	if err != nil {
+		fmt.Printf("Error: Unable to open edgyjson directory\n")
+		return
+	}
+	defer edgyJsonDir.Close()
+
+	var trans Transformer
+	_, fe, err := NewFilterExpressionParser("(int>equals10000) AND (int<>1000000) OR (float IS NOT NULL)")
+	expr, err := fe.OutputExpression()
+	assert.Nil(err)
+	matchDef := trans.Transform([]Expression{expr})
+	assert.NotNil(matchDef)
+	m := NewFastMatcher(matchDef)
+
+	edgyJsonList, _ := edgyJsonDir.Readdirnames(0)
+	for _, name := range edgyJsonList {
+		fileName := fmt.Sprintf("%s/%s", dirPath, name)
+		byteSlice, err := ioutil.ReadFile(fileName)
+		if err != nil {
+			fmt.Printf("Error: Unable to read %v\n", fileName)
+			continue
+		}
+
+		matched, err := m.Match(byteSlice)
+		assert.Nil(err)
+		assert.False(matched)
+		m.Reset()
+	}
+}