Fix remaining tokens that were not identifiers but should

Signed-off-by: Juanjo Alvarez <[email protected]>

Author: Juanjo Alvarez

driver/normalizer/normalizer.go

@@ -60,7 +60,23 @@ func funcDefMap(typ string, async bool) Mapping {
 	))
 }
 
+func tokenIsIdentifier(typ, tokenKey string, roles... role.Role) Mapping {
+	return AnnotateType(typ, MapObj(
+		Fields{
+			{Name: tokenKey, Op: Var("name")},
+		},
+		Fields{
+			{Name: tokenKey, Op: UASTType(uast.Identifier{}, Obj{
+				"Name": Var("name"),
+			})},
+		}),
+		roles...)
+}
+
 var Normalizers = []Mapping{
+
+	tokenIsIdentifier("keyword", "arg", role.Name),
+
 	MapSemantic("Str", uast.String{}, MapObj(
 		Obj{
 			"s": Var("val"),

fixtures/functioncalls.py.sem.uast

@@ -260,10 +260,12 @@
             },
             keywords: [
                { '@type': "keyword",
-                  '@token': "b",
                   '@role': [Argument, Call, Function, Name],
                   '@pos': { '@type': "uast:Positions",
                   },
+                  '@token': { '@type': "uast:Identifier",
+                     Name: "b",
+                  },
                   value: { '@type': "Num",
                      '@token': 1,
                      '@role': [Argument, Expression, Literal, Number, Primitive, Value],
@@ -282,10 +284,12 @@
                   },
                },
                { '@type': "keyword",
-                  '@token': "c",
                   '@role': [Argument, Call, Function, Name],
                   '@pos': { '@type': "uast:Positions",
                   },
+                  '@token': { '@type': "uast:Identifier",
+                     Name: "c",
+                  },
                   value: { '@type': "Num",
                      '@token': 2,
                      '@role': [Argument, Expression, Literal, Number, Primitive, Value],
@@ -444,7 +448,9 @@
                   '@role': [Argument, Call, Function, Name],
                   '@pos': { '@type': "uast:Positions",
                   },
-                  '@token': ~,
+                  '@token': { '@type': "uast:Identifier",
+                     Name: ~,
+                  },
                   value: { '@type': "uast:Identifier",
                      '@role': [Argument, Value],
                      '@pos': { '@type': "uast:Positions",

fixtures/issue62_b.py.sem.uast

@@ -1498,10 +1498,12 @@
                      },
                      keywords: [
                         { '@type': "keyword",
-                           '@token': "linguist",
                            '@role': [Argument, Call, Function, Name],
                            '@pos': { '@type': "uast:Positions",
                            },
+                           '@token': { '@type': "uast:Identifier",
+                              Name: "linguist",
+                           },
                            value: { '@type': "uast:String",
                               '@role': [Argument, Value],
                               '@pos': { '@type': "uast:Positions",
@@ -1521,10 +1523,12 @@
                            },
                         },
                         { '@type': "keyword",
-                           '@token': "bblfsh_endpoint",
                            '@role': [Argument, Call, Function, Name],
                            '@pos': { '@type': "uast:Positions",
                            },
+                           '@token': { '@type': "uast:Identifier",
+                              Name: "bblfsh_endpoint",
+                           },
                            value: { '@type': "uast:String",
                               '@role': [Argument, Value],
                               '@pos': { '@type': "uast:Positions",