Checkpoint with the C# like transform

Signed-off-by: Juanjo Alvarez <[email protected]>

Author: Juanjo Alvarez

Gopkg.lock

@@ -3,7 +3,7 @@
 
 [[constraint]]
   name = "gopkg.in/bblfsh/sdk.v2"
-  version = "2.13.4"
+  version = "2.14.x"
 
 [prune]
   go-tests = true

Gopkg.toml

@@ -1,12 +1,25 @@
 package normalizer
 
 import (
+	"errors"
+
 	"gopkg.in/bblfsh/sdk.v2/uast"
+	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
 	"gopkg.in/bblfsh/sdk.v2/uast/role"
 	. "gopkg.in/bblfsh/sdk.v2/uast/transformer"
 )
 
 var Preprocess = Transformers([][]Transformer{
+	{Mappings(
+		// Move the Leading/TrailingTrivia outside of nodes.
+		//
+		// This cannot be inside Normalizers because it should precede any
+		// other transformation.
+		Map(
+			opMoveCommentsAnns{Var("group")},
+			Check(Has{uast.KeyType: String(typeGroup)}, Var("group")),
+		),
+	)},
 	{Mappings(Preprocessors...)},
 }...)
 
@@ -37,13 +50,19 @@ func funcDefMap(typ string, async bool) Mapping {
 				uast.KeyPos:  Var("_pos"),
 				uast.KeyType: Var("_type"),
 			},
+			// Will be filled only if there is a Python3 type annotation
+			"returns":        Var("returns"),
+			"decorator_list": Var("func_decorators"),
 		},
 		Obj{
 			"Nodes": Arr(
 				Obj{
 					// FIXME: generator=true if it uses yield anywhere in the body
 					"async": Bool(async),
 				},
+				Obj{
+					"decorators": Var("func_decorators"),
+				},
 				UASTType(uast.Alias{}, Obj{
 					// FIXME: can't call identifierWithPos because it would take the position of the
 					// function node that is not exactly the same as the position of the function name
@@ -53,6 +72,7 @@ func funcDefMap(typ string, async bool) Mapping {
 					"Node": UASTType(uast.Function{}, Obj{
 						"Type": UASTType(uast.FunctionType{}, Obj{
 							"Arguments": Var("arguments"),
+							"Returns":   Var("returns"),
 						}),
 						"Body": UASTType(uast.Block{}, Obj{
 							"Statements": Var("body"),
@@ -89,17 +109,146 @@ func mapStr(nativeType string) Mapping {
 			{Name: "boxed_value", Op: UASTType(uast.String{}, Obj{
 				uast.KeyPos: Var("pos_"),
 				"Value":     Var("s"),
-				//"Format":    String(""),
 			})},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
 		}),
 	)
 }
 
+var (
+	typeGroup     = uast.TypeOf(uast.Group{})
+	typeFuncGroup = uast.TypeOf(uast.FunctionGroup{})
+)
+
+// Similar (but not equal) to:
+// https://github.com/bblfsh/csharp-driver/blob/master/driver/normalizer/normalizer.go#L827
+// Should be removed once we have some generic solution in the SDK
+type opMoveCommentsAnns struct {
+	sub Op
+}
+
+func (op opMoveCommentsAnns) Kinds() nodes.Kind {
+	return nodes.KindObject
+}
+
+func (op opMoveCommentsAnns) Check(st *State, n nodes.Node) (bool, error) {
+	obj, ok := n.(nodes.Object)
+	if !ok {
+		return false, nil
+	}
+	modified := false
+	noops_prevs, ok1 := obj["noops_previous"].(nodes.Array)
+	noops_same, ok2 := obj["noops_sameline"].(nodes.Array)
+	//decorators, ok3 := obj["decorator_list"].(nodes.Array)
+
+	if ok1 || ok2 {
+		// we saved the comments and decorators, remove from them from this node
+		obj = obj.CloneObject()
+		modified = true
+		delete(obj, "noops_previous")
+		delete(obj, "noops_sameline")
+		//delete(obj, "decorator_list")
+	}
+
+	if len(noops_prevs) == 0 || len(noops_same) == 0 {
+		if !modified {
+			return false, nil
+		}
+		return op.sub.Check(st, obj)
+	}
+
+	arr := make(nodes.Array, 0, len(noops_prevs) + 1 + len(noops_same))
+	arr = append(arr, noops_prevs...)
+	arr = append(arr, obj)
+	arr = append(arr, noops_same)
+	group, err := uast.ToNode(uast.Group{})
+	if err != nil {
+		return false, err
+	}
+
+	obj = group.(nodes.Object)
+	obj["Nodes"] = arr
+	return op.sub.Check(st, obj)
+}
+
+func (op opMoveCommentsAnns) Construct(st *State, n nodes.Node) (nodes.Node, error) {
+	// TODO(dennwc): implement when we will need a reversal
+	//				 see https://github.com/bblfsh/sdk/issues/355
+	return op.sub.Construct(st, n)
+}
+
+// opMergeGroups finds the uast:Group nodes and merges them into a child
+// uast:FunctionGroup, if it exists.
+//
+// This transform is necessary because opMoveTrivias will wrap all nodes that contain trivia
+// into a Group node, and the same will happen with MethodDeclaration nodes. But according
+// to a UAST schema defined in SDK, the comments (trivia) should be directly inside the
+// FunctionGroup node that wraps functions in Semantic mode.
+type opMergeGroups struct {
+	sub Op
+}
+
+func (op opMergeGroups) Kinds() nodes.Kind {
+	return nodes.KindObject
+}
+
+
+// firstWithType returns an index of the first node type of which matches the filter function.
+func firstWithType(arr nodes.Array, fnc func(typ string) bool) int {
+	for i, sub := range arr {
+		if fnc(uast.TypeOf(sub)) {
+			return i
+		}
+	}
+	return -1
+}
+
+// Check tests if the current node is uast:Group and if it contains a uast:FunctionGroup
+// node, it will remove the current node and merge other children into the FunctionGroup.
+func (op opMergeGroups) Check(st *State, n nodes.Node) (bool, error) {
+	group, ok := n.(nodes.Object)
+	if !ok || uast.TypeOf(group) != typeGroup {
+		return false, nil
+	}
+	arr, ok := group["Nodes"].(nodes.Array)
+	if !ok {
+		return false, errors.New("expected an array in Group.Nodes")
+	}
+	ind := firstWithType(arr, func(typ string) bool {
+		return typ == typeFuncGroup
+	})
+	if ind < 0 {
+		return false, nil
+	}
+	leading := arr[:ind]
+	fgroup := arr[ind].(nodes.Object)
+	trailing := arr[ind+1:]
+
+	arr, ok = fgroup["Nodes"].(nodes.Array)
+	if !ok {
+		return false, errors.New("expected an array in Group.Nodes")
+	}
+	out := make(nodes.Array, 0, len(leading)+len(arr)+len(trailing))
+	out = append(out, leading...)
+	out = append(out, arr...)
+	out = append(out, trailing...)
+
+	fgroup = fgroup.CloneObject()
+	fgroup["Nodes"] = out
+
+	return op.sub.Check(st, fgroup)
+}
+
+func (op opMergeGroups) Construct(st *State, n nodes.Node) (nodes.Node, error) {
+	// TODO(dennwc): implement when we will need a reversal
+	//				 see https://github.com/bblfsh/sdk/issues/355
+	return op.sub.Construct(st, n)
+}
+
 var Normalizers = []Mapping{
 
-	// Box Names, Strings, and Bools into a "BoxedFoo" moving the real node to the
+	// Box Names, Strings, Attributes, ImportFroms and Bools into a "BoxedFoo" moving the real node to the
 	// "value" property and keeping the comments in the parent (if not, comments would be lost
 	// when promoting the objects)
 	Map(
@@ -121,13 +270,10 @@ var Normalizers = []Mapping{
 		}),
 	),
 
-	// TODO: uncomment after SDK 2.13.x update
-	//  (upgrade currently blocked by: https://github.com/bblfsh/sdk/issues/353)
 	Map(
 		Part("_", Fields{
 			{Name: uast.KeyType, Op: String("BoolLiteral")},
 			{Name: uast.KeyPos, Op: Var("pos_")},
-			//{Name: "LiteralValue", Op: Var("lv")},
 			{Name: "value", Op: Var("lv")},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
@@ -136,13 +282,39 @@ var Normalizers = []Mapping{
 			{Name: uast.KeyType, Op: String("BoxedBoolLiteral")},
 			{Name: "boxed_value", Op: UASTType(uast.Bool{}, Obj{
 				uast.KeyPos: Var("pos_"),
+<<<<<<< HEAD
+				"Value":     Var("lv"),
+||||||| merged common ancestors
+				"Value": Var("lv"),
+=======
 				"Value":     Var("lv"),
 			})},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
 		}),
 	),
 
+	Map(
+		Part("_", Fields{
+			{Name: uast.KeyType, Op: String("Attribute")},
+			{Name: uast.KeyPos, Op: Var("pos_")},
+			{Name: "attr", Op: Var("aname")},
+			{Name: "ctx", Op: Any()},
+			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
+			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
+		}),
+		Part("_", Fields{
+			{Name: uast.KeyType, Op: String("BoxedAttribute")},
+			{Name: "boxed_value", Op: UASTType(uast.Identifier{}, Obj{
+				uast.KeyPos: Var("pos_"),
+				"Name":      Var("aname"),
+>>>>>>> Checkpoint with the C# like transform
+			})},
+			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
+			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
+		}),
+	),
+
 	mapStr("Bytes"),
 	mapStr("Str"),
 	mapStr("StringLiteral"),
@@ -174,15 +346,11 @@ var Normalizers = []Mapping{
 		}),
 		role.Name),
 
-	MapSemantic("Attribute", uast.Identifier{}, MapObj(
-		Obj{"attr": Var("name")},
-		Obj{"Name": Var("name")},
-	)),
-
 	MapSemantic("arg", uast.Argument{}, MapObj(
 		Obj{
 			uast.KeyToken: Var("name"),
 			"default":     Var("init"),
+			"ctx":         Any(),
 		},
 		Obj{
 			"Name": identifierWithPos("name"),
@@ -193,6 +361,7 @@ var Normalizers = []Mapping{
 	MapSemantic("arg", uast.Argument{}, MapObj(
 		Obj{
 			uast.KeyToken: Var("name"),
+			"ctx":         Any(),
 		},
 		Obj{
 			"Name": identifierWithPos("name"),
@@ -203,6 +372,8 @@ var Normalizers = []Mapping{
 		Obj{
 			uast.KeyToken: Var("name"),
 			"default":     Var("init"),
+			// TODO: change this once we've a way to store other nodes on semantic objects
+			"annotation": Any(),
 		},
 		Obj{
 			"Init": Var("init"),
@@ -213,6 +384,8 @@ var Normalizers = []Mapping{
 	MapSemantic("vararg", uast.Argument{}, MapObj(
 		Obj{
 			uast.KeyToken: Var("name"),
+			// TODO: change this once we've a way to store other nodes on semantic objects
+			"annotation": Any(),
 		},
 		Obj{
 			"Name":     identifierWithPos("name"),
@@ -223,6 +396,8 @@ var Normalizers = []Mapping{
 	MapSemantic("kwarg", uast.Argument{}, MapObj(
 		Obj{
 			uast.KeyToken: Var("name"),
+			// TODO: change this once we've a way to store other nodes on semantic objects
+			"annotation": Any(),
 		},
 		Obj{
 			"Name": identifierWithPos("name"),
@@ -317,4 +492,10 @@ var Normalizers = []Mapping{
 			}),
 		},
 	)),
+
+	// Merge uast:Group with uast:FunctionGroup.
+	Map(
+		opMergeGroups{Var("group")},
+		Check(Has{uast.KeyType: String(uast.TypeOf(uast.FunctionGroup{}))}, Var("group")),
+	),
 }