diff --git a/Tmain/json-output-format.d/stdout-expected.txt b/Tmain/json-output-format.d/stdout-expected.txt index b8e96b9d5e..250c277e7c 100644 --- a/Tmain/json-output-format.d/stdout-expected.txt +++ b/Tmain/json-output-format.d/stdout-expected.txt @@ -75,7 +75,7 @@ {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "s,section", "pattern": "sections"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "t,title", "pattern": "titles"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "I,namespace", "pattern": "name referring a module defined in other file"} -{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "name referring a class/variable/function/module defined in other module"} +{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "unknwon name"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "c,class", "pattern": "classes"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "f,function", "pattern": "functions"} {"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "i,module", "pattern": "modules"} @@ -97,6 +97,7 @@ {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "C", "kindName": "struct", "path": "foreigndecl", "pattern": "declared in foreign languages"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Go", "kindName": "package", "path": "imported", "pattern": "imported package"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Go", "kindName": "unknown", "path": "receiverType", "pattern": "receiver type"} +{"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "class", "path": "super", "pattern": "super class"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "imported", "pattern": "imported modules"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "indirectlyImported", "pattern": "module imported in alternative name"} {"_type": "ptag", "name": "TAG_ROLE_DESCRIPTION", "parserName": "Python", "kindName": "module", "path": "namespace", "pattern": "namespace from where classes/variables/functions are imported"} diff --git a/Tmain/list-roles.d/stdout-expected.txt b/Tmain/list-roles.d/stdout-expected.txt index 32ec05dc78..cba7d3b7bd 100644 --- a/Tmain/list-roles.d/stdout-expected.txt +++ b/Tmain/list-roles.d/stdout-expected.txt @@ -91,6 +91,8 @@ Protobuf D/protodef imported on imported Protobuf m/message extension on extending the message Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow +Python c/class super on super class Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported @@ -224,6 +226,8 @@ Protobuf D/protodef imported on imported Protobuf m/message extension on extending the message Python Y/unknown imported on imported from the other module Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name +Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow +Python c/class super on super class Python i/module imported on imported modules Python i/module indirectlyImported on module imported in alternative name Python i/module namespace on namespace from where classes/variables/functions are imported diff --git a/Units/parser-python.r/reftag.d/args.ctags b/Units/parser-python.r/reftag.d/args.ctags new file mode 100644 index 0000000000..6f1b8e0f7d --- /dev/null +++ b/Units/parser-python.r/reftag.d/args.ctags @@ -0,0 +1,5 @@ +--sort=no +--extras=+r +--fields=+r +--kinds-Python=* +--roles-Python.{unknown}=+{ref} diff --git a/Units/parser-python.r/reftag.d/expected.tags b/Units/parser-python.r/reftag.d/expected.tags new file mode 100644 index 0000000000..8cd3e61fc0 --- /dev/null +++ b/Units/parser-python.r/reftag.d/expected.tags @@ -0,0 +1,20 @@ +a.b input.py /^import a.b$/;" i roles:imported +object input.py /^class X(object):$/;" c roles:super +X input.py /^class X(object):$/;" c roles:def +__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def +self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def +n input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def +n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref +n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref +val input.py /^ def val(self):$/;" m class:X roles:def +self input.py /^ def val(self):$/;" z member:X.val file: roles:def +n input.py /^ return self.n$/;" Y member:X.val roles:ref +one input.py /^def one():$/;" f roles:def +X input.py /^ return X(1)$/;" Y function:one roles:ref +two input.py /^def two():$/;" f roles:def +X input.py /^ return X(2)$/;" Y function:two roles:ref +print input.py /^print (one().val() + two().val())$/;" Y roles:ref +one input.py /^print (one().val() + two().val())$/;" Y roles:ref +val input.py /^print (one().val() + two().val())$/;" Y roles:ref +two input.py /^print (one().val() + two().val())$/;" Y roles:ref +val input.py /^print (one().val() + two().val())$/;" Y roles:ref diff --git a/Units/parser-python.r/reftag.d/input.py b/Units/parser-python.r/reftag.d/input.py new file mode 100644 index 0000000000..0136a53227 --- /dev/null +++ b/Units/parser-python.r/reftag.d/input.py @@ -0,0 +1,18 @@ +import a.b + +class X(object): + def __init__(self, n): + self.n = n + + def val(self): + return self.n + +def one(): + return X(1) + +def two(): + return X(2) + +print (one().val() + two().val()) + + diff --git a/Units/parser-python.r/simple.py.d/expected.tags b/Units/parser-python.r/simple.py.d/expected.tags index 6379eef842..a7891e5344 100644 --- a/Units/parser-python.r/simple.py.d/expected.tags +++ b/Units/parser-python.r/simple.py.d/expected.tags @@ -37,9 +37,13 @@ this input.py /^ @blah class this is seen???$/;" kind:class l _test.ignored_function.more_nesting.deeply_nested.even_more.this input.py /^ @blah class this is seen???$/;" kind:class line:41 language:Python scope:member:_test.ignored_function.more_nesting.deeply_nested.even_more file: inherits: access:private roles:def extras:qualified end:42 decorators:blah this input.py /^ @bleh def this also? good!$/;" kind:member line:42 language:Python scope:class:_test.ignored_function.more_nesting.deeply_nested.even_more.this access:public roles:def end:42 decorators:bleh _test.ignored_function.more_nesting.deeply_nested.even_more.this.this input.py /^ @bleh def this also? good!$/;" kind:member line:42 language:Python scope:class:_test.ignored_function.more_nesting.deeply_nested.even_more.this access:public roles:def extras:qualified end:42 decorators:bleh +one input.py /^class two (one):$/;" kind:class line:46 language:Python roles:super extras:reference two input.py /^class two (one):$/;" kind:class line:46 language:Python inherits:one access:public roles:def end:48 only input.py /^ def only(arg):$/;" kind:member line:48 language:Python scope:class:two access:public signature:(arg) roles:def end:48 two.only input.py /^ def only(arg):$/;" kind:member line:48 language:Python scope:class:two access:public signature:(arg) roles:def extras:qualified end:48 +A input.py /^(A, B,$/;" kind:class line:53 language:Python roles:super extras:reference +B input.py /^(A, B,$/;" kind:class line:53 language:Python roles:super extras:reference +C input.py /^C):$/;" kind:class line:54 language:Python roles:super extras:reference three input.py /^three\\$/;" kind:class line:52 language:Python inherits:A, B, C access:public roles:def end:54 foo input.py /^foo($/;" kind:function line:57 language:Python access:public signature:( x , y, z) roles:def end:60 input.py input.py 1;" kind:file line:1 language:Python roles:def extras:inputFile end:60 diff --git a/parsers/python.c b/parsers/python.c index e59e60836b..693a6b1cbd 100644 --- a/parsers/python.c +++ b/parsers/python.c @@ -44,6 +44,11 @@ enum { KEYWORD_lambda, KEYWORD_pass, KEYWORD_return, + + /* Used only in readTokenFullNoRefTag to represent the identifiers + that should not be tagged as reference tags. */ + KEYWORD___noreftag_id, + KEYWORD_REST }; typedef int keywordId; /* to allow KEYWORD_NONE */ @@ -96,10 +101,15 @@ typedef enum { } pythonModuleRole; typedef enum { + PYTHON_UNKNOWN_REFERENCED, PYTHON_UNKNOWN_IMPORTED, PYTHON_UNKNOWN_INDIRECTLY_IMPORTED, } pythonUnknownRole; +typedef enum { + PYTHON_CLASS_SUPERCLASS, +} pythonClassRole; + /* Roles related to `import' * ========================== * import X X = (kind:module, role:imported) @@ -128,20 +138,26 @@ static roleDefinition PythonModuleRoles [] = { }; static roleDefinition PythonUnknownRoles [] = { + { false,"ref", "(EXPERIMENTAL)referenced anyhow" }, { true, "imported", "imported from the other module" }, { true, "indirectlyImported", "classes/variables/functions/modules imported in alternative name" }, }; +static roleDefinition PythonClassRoles [] = { + { true, "super", "super class" }, +}; + static kindDefinition PythonKinds[COUNT_KIND] = { - {true, 'c', "class", "classes"}, + {true, 'c', "class", "classes", + .referenceOnly = false, ATTACH_ROLES(PythonClassRoles) }, {true, 'f', "function", "functions"}, {true, 'm', "member", "class members"}, {true, 'v', "variable", "variables"}, {true, 'I', "namespace", "name referring a module defined in other file"}, {true, 'i', "module", "modules", .referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)}, - {true, 'Y', "unknown", "name referring a class/variable/function/module defined in other module", + {true, 'Y', "unknown", "unknwon name", .referenceOnly = false, ATTACH_ROLES(PythonUnknownRoles)}, {false, 'z', "parameter", "function parameters" }, {false, 'l', "local", "local variables" }, @@ -163,6 +179,7 @@ static const keywordTable PythonKeywordTable[] = { { "lambda", KEYWORD_lambda }, { "pass", KEYWORD_pass }, { "return", KEYWORD_return }, + { "self", KEYWORD___noreftag_id }, }; /* Taken from https://docs.python.org/3/reference/lexical_analysis.html#keywords */ @@ -201,6 +218,7 @@ typedef struct { int indent; unsigned long lineNumber; MIOPos filePosition; + int reftag; } tokenInfo; struct pythonNestingLevelUserData { @@ -244,13 +262,23 @@ static accessType accessFromIdentifier (const vString *const ident, return ACCESS_PROTECTED; } -static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token, +static void useTokenAsPartOfAnotherTag (tokenInfo *const token) +{ + if (token->reftag == CORK_NIL) + return; + + markCorkEntryAsPlaceholder (token->reftag, true); + token->reftag = CORK_NIL; +} + +static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token, const pythonKind kind) { accessType access; int parentKind = -1; NestingLevel *nl; + useTokenAsPartOfAnotherTag(token); initTagEntry (e, vStringValue (token->string), kind); updateTagLine (e, token->lineNumber, token->filePosition); @@ -283,7 +311,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token e->isFileScope = true; } -static int makeClassTag (const tokenInfo *const token, +static int makeClassTag (tokenInfo *const token, const vString *const inheritance, const vString *const decorators) { @@ -306,7 +334,7 @@ static int makeClassTag (const tokenInfo *const token, return CORK_NIL; } -static int makeFunctionTag (const tokenInfo *const token, +static int makeFunctionTag (tokenInfo *const token, const vString *const arglist, const vString *const decorators) { @@ -330,7 +358,7 @@ static int makeFunctionTag (const tokenInfo *const token, return CORK_NIL; } -static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind) +static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind) { if (PythonKinds[kind].enabled) { @@ -343,7 +371,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k return CORK_NIL; } -static int makeSimplePythonRefTag (const tokenInfo *const token, +static int makeSimplePythonRefTag (tokenInfo *const token, const vString *const altName, pythonKind const kind, int roleIndex, xtagType xtag) @@ -353,6 +381,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token, { tagEntryInfo e; + useTokenAsPartOfAnotherTag(token); initRefTagEntry (&e, vStringValue (altName ? altName : token->string), kind, roleIndex); @@ -391,6 +420,7 @@ static void clearPoolToken (void *data) token->lineNumber = getInputLineNumber (); token->filePosition = getInputFilePosition (); vStringClear (token->string); + token->reftag = CORK_NIL; } static void copyToken (tokenInfo *const dest, const tokenInfo *const src) @@ -401,6 +431,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src) dest->keyword = src->keyword; dest->indent = src->indent; vStringCopy(dest->string, src->string); + dest->reftag = src->reftag; } /* Skip a single or double quoted string. */ @@ -477,7 +508,7 @@ static void ungetToken (tokenInfo *const token) copyToken (NextToken, token); } -static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces, bool *noReftagId) { int c; int n; @@ -664,11 +695,17 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) } else { + *noReftagId = false; /* FIXME: handle U, B, R and F string prefixes? */ readIdentifier (token->string, c); token->keyword = lookupKeyword (vStringValue (token->string), Lang_python); if (token->keyword == KEYWORD_NONE) token->type = TOKEN_IDENTIFIER; + else if (token->keyword == KEYWORD___noreftag_id) + { + token->type = TOKEN_IDENTIFIER; + *noReftagId = true; + } else token->type = TOKEN_KEYWORD; } @@ -692,6 +729,38 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) } } +static void readTokenFull (tokenInfo *const token, bool inclWhitespaces) +{ + bool noReftagId; + readTokenFullNoRefTag (token, inclWhitespaces, &noReftagId); + + if (token->type == TOKEN_IDENTIFIER + && (!noReftagId) + /* Don't make a ref tag for a number. */ + && (vStringLength (token->string) > 0 && + !isdigit ((unsigned char)vStringChar (token->string, 0))) + && PythonKinds[K_UNKNOWN].enabled + && PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled) + { + const bool in_subparser = (Lang_python != getInputLanguage ()); + if (in_subparser) + pushLanguage (Lang_python); + + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (token->string), + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + e.lineNumber = token->lineNumber; + e.filePosition = token->filePosition; + NestingLevel *nl = nestingLevelsGetCurrent (PythonNestingLevels); + if (nl) + e.extensionFields.scopeIndex = nl->corkIndex; + token->reftag = makeTagEntry (&e); + + if (in_subparser) + popLanguage (); + } +} + static void readToken (tokenInfo *const token) { readTokenFull (token, false); @@ -786,6 +855,10 @@ static void readQualifiedName (tokenInfo *const nameToken) vString *qualifiedName = vStringNew (); tokenInfo *token = newToken (); + unsigned long lineNumber = nameToken->lineNumber; + MIOPos filePosition = nameToken->filePosition; + + useTokenAsPartOfAnotherTag (nameToken); while (nameToken->type == TOKEN_IDENTIFIER || nameToken->type == '.') { @@ -793,6 +866,7 @@ static void readQualifiedName (tokenInfo *const nameToken) copyToken (token, nameToken); readToken (nameToken); + useTokenAsPartOfAnotherTag (nameToken); } /* put the last, non-matching, token back */ ungetToken (nameToken); @@ -803,6 +877,13 @@ static void readQualifiedName (tokenInfo *const nameToken) deleteToken (token); vStringDelete (qualifiedName); + + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (nameToken->string), + K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED); + e.lineNumber = lineNumber; + e.filePosition = filePosition; + nameToken->reftag = makeTagEntry (&e); } } @@ -981,14 +1062,53 @@ static void deleteTypedParam (struct typedParam *p) eFree (p); } -static void parseArglist (tokenInfo *const token, const int kind, +static void parseInheritanceList (tokenInfo *const token, + vString *const inneritanceList) +{ + tokenInfo *lastToken = newToken (); + + do + { + copyToken (lastToken, token); + readTokenFull (token, true); + if (lastToken->type == TOKEN_IDENTIFIER + && (token->type == ',' || token->type == ')')) + { + if (lastToken->reftag == CORK_NIL) + { + tagEntryInfo e; + initRefTagEntry(&e, vStringValue (lastToken->string), + K_CLASS, PYTHON_CLASS_SUPERCLASS); + e.lineNumber = lastToken->lineNumber; + e.filePosition = lastToken->filePosition; + lastToken->reftag = makeTagEntry (&e); + } + else + { + tagEntryInfo *e = getEntryInCorkQueue (lastToken->reftag); + if (e) + { + clearRoles(e); + e->kindIndex = K_CLASS; + assignRole(e, PYTHON_CLASS_SUPERCLASS); + } + } + } + if (token->type != ')') + reprCat (inneritanceList, token); + } + while (token->type != TOKEN_EOF && token->type != ')'); + + deleteToken (lastToken); +} + +static void parseArglist (tokenInfo *const token, vString *const arglist, ptrArray *const parameters) { int prevTokenType = token->type; int depth = 1; - if (kind != K_CLASS) - reprCat (arglist, token); + reprCat (arglist, token); do { @@ -1001,8 +1121,7 @@ static void parseArglist (tokenInfo *const token, const int kind, } readTokenFull (token, true); - if (kind != K_CLASS || token->type != ')' || depth > 1) - reprCat (arglist, token); + reprCat (arglist, token); if (token->type == '(' || token->type == '[' || @@ -1012,7 +1131,7 @@ static void parseArglist (tokenInfo *const token, const int kind, token->type == ']' || token->type == '}') depth --; - else if (kind != K_CLASS && depth == 1 && + else if (depth == 1 && token->type == TOKEN_IDENTIFIER && (prevTokenType == '(' || prevTokenType == ',') && PythonKinds[K_PARAMETER].enabled) @@ -1022,6 +1141,7 @@ static void parseArglist (tokenInfo *const token, const int kind, struct typedParam *parameter; parameterName = newToken (); + useTokenAsPartOfAnotherTag (token); copyToken (parameterName, token); parameterType = parseParamTypeAnnotation (token, arglist); @@ -1032,8 +1152,8 @@ static void parseArglist (tokenInfo *const token, const int kind, while (token->type != TOKEN_EOF && depth > 0); } -static void parseCArglist (tokenInfo *const token, const int kind, - vString *const arglist, ptrArray *const parameters) +static void parseCArglist (tokenInfo *const token, + vString *const arglist, ptrArray *const parameters) { int depth = 1; tokenInfo *pname = newToken (); @@ -1172,12 +1292,14 @@ static bool parseClassOrDef (tokenInfo *const token, if (token->type == '(') { arglist = vStringNew (); - parameters = ptrArrayNew ((ptrArrayDeleteFunc)deleteTypedParam); + parameters = (kind == K_CLASS)? NULL: ptrArrayNew ((ptrArrayDeleteFunc)deleteTypedParam); - if (isCDef && kind != K_CLASS) - parseCArglist (token, kind, arglist, parameters); + if (kind == K_CLASS) + parseInheritanceList (token, arglist); + else if (isCDef) + parseCArglist (token, arglist, parameters); else - parseArglist (token, kind, arglist, parameters); + parseArglist (token, arglist, parameters); } if (kind == K_CLASS) @@ -1488,7 +1610,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind) do { - const tokenInfo *const nameToken = nameTokens[i]; + tokenInfo *const nameToken = nameTokens[i]; vString **type = &(nameTypes[i++]); readToken (token);