Merge pull request #85 from juanjux/feature/ast_split

Feature/ast split

Author: juanjux

Dockerfile.tpl

@@ -3,7 +3,7 @@ MAINTAINER source{d}
 
 ARG DEVDEPS=native/dev_deps
 ARG CONTAINER_DEVDEPS=/tmp/dev_deps
-ARG PYDETECTOR_VER=0.13.0
+ARG PYDETECTOR_VER=0.14.0
 
 RUN apk add --no-cache --update python python3 py-pip py2-pip git
 

driver/normalizer/parser.go

@@ -23,36 +23,52 @@ var ToNoder = &native.ObjectToNoder{
 		"noop_line":    true, // Comment/Noop (non significative whitespace)
 	},
 	SyntheticTokens: map[string]string{
-		"Print":     "print",
-		"Ellipsis":  "PythonEllipsisOperator",
 		"Add":       "+",
-		"Sub":       "-",
-		"Mult":      "*",
-		"Div":       "/",
-		"FloorDiv":  "//",
-		"Mod":       "%%",
-		"Pow":       "**",
+		"Assert":    "assert",
 		"AugAssign": "?=",
 		"BitAnd":    "&",
 		"BitOr":     "|",
 		"BitXor":    "^",
-		"LShift":    "<<",
-		"RShift":    ">>",
-		"Eq":        "==",
-		"NotEq":     "!=",
-		"Not":       "!",
-		"Lt":        "<",
-		"LtE":       "<=",
-		"Gt":        ">",
-		"GtE":       ">=",
-		"Is":        "is",
-		"IsNot":     "not is",
-		"In":        "in",
-		"NotIn":     "not in",
-		"UAdd":      "+",
-		"USub":      "-",
-		"Invert":    "~",
-		"Pass":      "pass",
+		"Break":     "break",
+		"Continue": "continue",
+		"Delete":   "delete",
+		"Div":      "/",
+		"Ellipsis": "...",
+		"Eq":       "==",
+		"False":    "false",
+		"For":      "for",
+		"FloorDiv": "//",
+		"Global":   "global",
+		"Gt":       ">",
+		"GtE":      ">=",
+		"If":       "if",
+		"In":       "in",
+		"Invert":   "~",
+		"Is":       "is",
+		"IsNot":    "not is",
+		"LShift":   "<<",
+		"Lt":       "<",
+		"LtE":      "<=",
+		"Mod":      "%%",
+		"Mult":     "*",
+		"None":     "None",
+		"Nonlocal": "nonlocal",
+		"Not":      "!",
+		"NotEq":    "!=",
+		"NotIn":    "not in",
+		"Pass":     "pass",
+		"Pow":      "**",
+		"Print":    "print",
+		"Raise":    "raise",
+		"Return":   "return",
+		"RShift":   ">>",
+		"Sub":      "-",
+		"True":     "true",
+		"UAdd":     "+",
+		"USub":     "-",
+		"While":    "while",
+		"With":     "with",
+		"Yield":    "yield",
 	},
 	PromoteAllPropertyLists: false,
 	PromotedPropertyLists: map[string]map[string]bool{