fix python_tokenizer tok_name error (#9)

* fix python_tokenizer tok_name error

* remove import which is not required in python_tokenizer.py

* remove empty value tokens

Author: tushargr

tokenizer/python/python_tokenizer.py

@@ -1,21 +1,20 @@
 import parso
 from parso.python.tokenize import tokenize
-from parso.python.token import tok_name
 import json
-import sys 
+import sys
 
 file = open(sys.argv[1],'r')
 file_content = file.read()
 
 tokens = []
 
 for token in tokenize(file_content, version_info=(3,6)):
-	tmp = dict()
-	tmp["line"]=(token.start_pos)[0];
-	tmp["char"]=((token.start_pos)[1])+1;
-	tmp["type"]=str(tok_name[token.type])
-	tmp["value"]=str(token.string)
-	tokens.append(tmp)
+        if (str(token.string) != ""):
+                tmp = dict()
+                tmp["line"]=(token.start_pos)[0];
+                tmp["char"]=((token.start_pos)[1])+1;
+                tmp["type"]=((str(token.type))[10:]).strip(")")
+                tmp["value"]=str(token.string)
+                tokens.append(tmp)
 
 print ( json.dumps(tokens, indent=4, sort_keys=True) )
-