Goddamnyou unrolling, fixed a bug and test it

Author: zimmski

token/transform.go

@@ -73,7 +73,6 @@ func UnrollPointers(root Token) Token {
 	log.Debug("start unrolling pointers by cloning them")
 
 	parents := make(map[Token]Token)
-	changed := make(map[Token]struct{})
 
 	originals := make(map[Token]Token)
 	originalClones := make(map[Token]Token)
@@ -178,8 +177,6 @@ func UnrollPointers(root Token) Token {
 				if iTok.parent != nil {
 					log.Debugf("replace in (%p)%#v", iTok.parent.tok, iTok.parent.tok)
 
-					changed[iTok.parent.tok] = struct{}{}
-
 					switch tt := iTok.parent.tok.(type) {
 					case ForwardToken:
 						tt.InternalReplace(t, c)
@@ -230,7 +227,6 @@ func UnrollPointers(root Token) Token {
 			REMOVE:
 				for tt != nil {
 					delete(parents, tt.tok)
-					delete(changed, tt.tok)
 
 					switch l := tt.tok.(type) {
 					case ForwardToken:
@@ -291,23 +287,21 @@ func UnrollPointers(root Token) Token {
 		}
 	}
 
-	// we need to update some tokens with the same child to regenerate clones
-	for child := range changed {
-		parent := parents[child]
-
-		if parent == nil {
-			continue
-		}
-
-		log.Debugf("update (%p)%#v with child (%p)%#v", parent, parent, child, child)
-
-		switch tt := parent.(type) {
+	// force regeneration of possible cloned tokens
+	WalkInternalTail(root, func(tok Token) error {
+		switch t := tok.(type) {
 		case ForwardToken:
-			tt.InternalReplace(child, child)
+			c := t.InternalGet()
+			t.InternalReplace(c, c)
 		case ListToken:
-			tt.InternalReplace(child, child)
+			for i := 0; i < t.InternalLen(); i++ {
+				c, _ := t.InternalGet(i)
+				t.InternalReplace(c, c)
+			}
 		}
-	}
+
+		return nil
+	})
 
 	log.Debug("finished unrolling")
 

token/transform_test.go

@@ -0,0 +1,54 @@
+package token_test
+
+import (
+	"testing"
+
+	. "github.com/zimmski/tavor/test/assert"
+
+	"github.com/zimmski/tavor/token"
+	"github.com/zimmski/tavor/token/lists"
+	"github.com/zimmski/tavor/token/primitives"
+)
+
+func TestUnrollPointers(t *testing.T) {
+	{
+		// Corner case: START = "group" +(START | "a")
+		// This should not leave any pointer in the internal as well as in the external representation
+
+		var tok *token.Token
+
+		p := primitives.NewEmptyPointer(tok)
+		s := lists.NewAll(
+			primitives.NewConstantString("group"),
+			lists.NewRepeat(
+				lists.NewOne(
+					p,
+					primitives.NewConstantString("a"),
+				),
+				1,
+				2,
+			),
+		)
+
+		err := p.Set(s)
+		Nil(t, err)
+
+		unrolled := token.UnrollPointers(s)
+
+		token.WalkInternal(unrolled, func(tok token.Token) error {
+			if _, ok := tok.(*primitives.Pointer); ok {
+				t.Fatalf("Found pointer in the internal structure %#v", tok)
+			}
+
+			return nil
+		})
+
+		token.Walk(unrolled, func(tok token.Token) error {
+			if _, ok := tok.(*primitives.Pointer); ok {
+				t.Fatalf("Found pointer in the external structure %#v", tok)
+			}
+
+			return nil
+		})
+	}
+}

token/walk.go

@@ -80,6 +80,33 @@ func WalkInternal(root Token, walkFunc func(tok Token) error) error {
 	return nil
 }
 
+// WalkInternalTail traverses a internal token graph beginning from the given token and calls for every newly visited token the given function after it has traversed all children.
+// A depth-first algorithm is used to traverse the graph. If the given walk function returns an error, the whole walk process ends by returning the error back to the caller
+func WalkInternalTail(root Token, walkFunc func(tok Token) error) error {
+	switch t := root.(type) {
+	case ForwardToken:
+		if v := t.InternalGet(); v != nil {
+			if err := WalkInternalTail(v, walkFunc); err != nil {
+				return err
+			}
+		}
+	case ListToken:
+		for i := 0; i < t.InternalLen(); i++ {
+			c, _ := t.InternalGet(i)
+
+			if err := WalkInternalTail(c, walkFunc); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := walkFunc(root); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ReleaseTokens traverses the token graph and calls Release for every release token
 func ReleaseTokens(root Token) {
 	_ = Walk(root, func(tok Token) error {