Skip to content

Commit 740360d

Browse files
masatakemasatake
committed
Python: extract reference tags
Signed-off-by: Masatake YAMATO <[email protected]>
1 parent c99454e commit 740360d

File tree

6 files changed

+115
-9
lines changed

6 files changed

+115
-9
lines changed

‎Tmain/json-output-format.d/stdout-expected.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "s,section", "pattern": "sections"}
7676
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "t,title", "pattern": "titles"}
7777
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "I,namespace", "pattern": "name referring a module defined in other file"}
78-
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "name referring a class\\/variable\\/function\\/module defined in other module"}
78+
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "unknwon name"}
7979
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "c,class", "pattern": "classes"}
8080
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "f,function", "pattern": "functions"}
8181
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "i,module", "pattern": "modules"}

‎Tmain/list-roles.d/stdout-expected.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ Protobuf D/protodef imported on imported
8181
Protobuf m/message extension on extending the message
8282
Python Y/unknown imported on imported from the other module
8383
Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
84+
Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow
8485
Python i/module imported on imported modules
8586
Python i/module indirectlyImported on module imported in alternative name
8687
Python i/module namespace on namespace from where classes/variables/functions are imported
@@ -199,6 +200,7 @@ Protobuf D/protodef imported on imported
199200
Protobuf m/message extension on extending the message
200201
Python Y/unknown imported on imported from the other module
201202
Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
203+
Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow
202204
Python i/module imported on imported modules
203205
Python i/module indirectlyImported on module imported in alternative name
204206
Python i/module namespace on namespace from where classes/variables/functions are imported

‎Units/parser-python.r/reftag.d/args.ctags‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
--sort=no
2+
--extras=+r
3+
--fields=+r
4+
--kinds-Python=*
5+
--roles-Python.{unknown}=+{ref}

‎Units/parser-python.r/reftag.d/expected.tags‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
a.b input.py /^import a.b$/;" i roles:imported
2+
object input.py /^class X(object):$/;" Y roles:ref
3+
X input.py /^class X(object):$/;" c roles:def
4+
__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def
5+
self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def
6+
n input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def
7+
self input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
8+
n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
9+
n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
10+
val input.py /^ def val(self):$/;" m class:X roles:def
11+
self input.py /^ def val(self):$/;" z member:X.val file: roles:def
12+
self input.py /^ return self.n$/;" Y member:X.val roles:ref
13+
n input.py /^ return self.n$/;" Y member:X.val roles:ref
14+
one input.py /^def one():$/;" f roles:def
15+
X input.py /^ return X(1)$/;" Y function:one roles:ref
16+
two input.py /^def two():$/;" f roles:def
17+
X input.py /^ return X(2)$/;" Y function:two roles:ref
18+
print input.py /^print (one().val() + two().val())$/;" Y roles:ref
19+
one input.py /^print (one().val() + two().val())$/;" Y roles:ref
20+
val input.py /^print (one().val() + two().val())$/;" Y roles:ref
21+
two input.py /^print (one().val() + two().val())$/;" Y roles:ref
22+
val input.py /^print (one().val() + two().val())$/;" Y roles:ref

‎Units/parser-python.r/reftag.d/input.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import a.b
2+
3+
class X(object):
4+
def __init__(self, n):
5+
self.n = n
6+
7+
def val(self):
8+
return self.n
9+
10+
def one():
11+
return X(1)
12+
13+
def two():
14+
return X(2)
15+
16+
print (one().val() + two().val())
17+
18+

‎parsers/python.c‎

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ typedef enum {
9696
} pythonModuleRole;
9797

9898
typedef enum {
99+
PYTHON_UNKNOWN_REFERENCED,
99100
PYTHON_UNKNOWN_IMPORTED,
100101
PYTHON_UNKNOWN_INDIRECTLY_IMPORTED,
101102
} pythonUnknownRole;
@@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = {
128129
};
129130

130131
static roleDefinition PythonUnknownRoles [] = {
132+
{ false,"ref", "(EXPERIMENTAL)referenced anyhow" },
131133
{ true, "imported", "imported from the other module" },
132134
{ true, "indirectlyImported",
133135
"classes/variables/functions/modules imported in alternative name" },
@@ -141,7 +143,7 @@ static kindDefinition PythonKinds[COUNT_KIND] = {
141143
{true, 'I', "namespace", "name referring a module defined in other file"},
142144
{true, 'i', "module", "modules",
143145
.referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)},
144-
{true, 'Y', "unknown", "name referring a class/variable/function/module defined in other module",
146+
{true, 'Y', "unknown", "unknwon name",
145147
.referenceOnly = false, ATTACH_ROLES(PythonUnknownRoles)},
146148
{false, 'z', "parameter", "function parameters" },
147149
{false, 'l', "local", "local variables" },
@@ -201,6 +203,7 @@ typedef struct {
201203
int indent;
202204
unsigned long lineNumber;
203205
MIOPos filePosition;
206+
int reftag;
204207
} tokenInfo;
205208

206209
struct pythonNestingLevelUserData {
@@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident,
244247
return ACCESS_PROTECTED;
245248
}
246249

247-
static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token,
250+
static void useTokenAsPartOfAnotherTag (tokenInfo *const token)
251+
{
252+
if (token->reftag == CORK_NIL)
253+
return;
254+
255+
markCorkEntryAsPlaceholder (token->reftag, true);
256+
token->reftag = CORK_NIL;
257+
}
258+
259+
static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token,
248260
const pythonKind kind)
249261
{
250262
accessType access;
251263
int parentKind = -1;
252264
NestingLevel *nl;
253265

266+
useTokenAsPartOfAnotherTag(token);
254267
initTagEntry (e, vStringValue (token->string), kind);
255268

256269
e->lineNumber = token->lineNumber;
@@ -284,7 +297,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token
284297
e->isFileScope = true;
285298
}
286299

287-
static int makeClassTag (const tokenInfo *const token,
300+
static int makeClassTag (tokenInfo *const token,
288301
const vString *const inheritance,
289302
const vString *const decorators)
290303
{
@@ -307,7 +320,7 @@ static int makeClassTag (const tokenInfo *const token,
307320
return CORK_NIL;
308321
}
309322

310-
static int makeFunctionTag (const tokenInfo *const token,
323+
static int makeFunctionTag (tokenInfo *const token,
311324
const vString *const arglist,
312325
const vString *const decorators)
313326
{
@@ -331,7 +344,7 @@ static int makeFunctionTag (const tokenInfo *const token,
331344
return CORK_NIL;
332345
}
333346

334-
static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind)
347+
static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind)
335348
{
336349
if (PythonKinds[kind].enabled)
337350
{
@@ -344,7 +357,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k
344357
return CORK_NIL;
345358
}
346359

347-
static int makeSimplePythonRefTag (const tokenInfo *const token,
360+
static int makeSimplePythonRefTag (tokenInfo *const token,
348361
const vString *const altName,
349362
pythonKind const kind,
350363
int roleIndex, xtagType xtag)
@@ -354,6 +367,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token,
354367
{
355368
tagEntryInfo e;
356369

370+
useTokenAsPartOfAnotherTag(token);
357371
initRefTagEntry (&e, vStringValue (altName ? altName : token->string),
358372
kind, roleIndex);
359373

@@ -393,6 +407,7 @@ static void clearPoolToken (void *data)
393407
token->lineNumber = getInputLineNumber ();
394408
token->filePosition = getInputFilePosition ();
395409
vStringClear (token->string);
410+
token->reftag = CORK_NIL;
396411
}
397412

398413
static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
@@ -403,6 +418,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
403418
dest->keyword = src->keyword;
404419
dest->indent = src->indent;
405420
vStringCopy(dest->string, src->string);
421+
dest->reftag = src->reftag;
406422
}
407423

408424
/* Skip a single or double quoted string. */
@@ -479,7 +495,7 @@ static void ungetToken (tokenInfo *const token)
479495
copyToken (NextToken, token);
480496
}
481497

482-
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
498+
static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces)
483499
{
484500
int c;
485501
int n;
@@ -694,6 +710,36 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
694710
}
695711
}
696712

713+
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
714+
{
715+
readTokenFullNoRefTag (token, inclWhitespaces);
716+
717+
if (token->type == TOKEN_IDENTIFIER
718+
/* Don't make a ref tag for a number. */
719+
&& (vStringLength(token->string) > 0 &&
720+
!isdigit((unsigned char)vStringChar(token->string, 0)))
721+
&& PythonKinds[K_UNKNOWN].enabled
722+
&& PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled)
723+
{
724+
const bool in_subparser = (Lang_python != getInputLanguage());
725+
if (in_subparser)
726+
pushLanguage(Lang_python);
727+
728+
tagEntryInfo e;
729+
initRefTagEntry(&e, vStringValue (token->string),
730+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
731+
e.lineNumber = token->lineNumber;
732+
e.filePosition = token->filePosition;
733+
NestingLevel *nl = nestingLevelsGetCurrent (PythonNestingLevels);
734+
if (nl)
735+
e.extensionFields.scopeIndex = nl->corkIndex;
736+
token->reftag = makeTagEntry (&e);
737+
738+
if (in_subparser)
739+
popLanguage();
740+
}
741+
}
742+
697743
static void readToken (tokenInfo *const token)
698744
{
699745
readTokenFull (token, false);
@@ -788,13 +834,18 @@ static void readQualifiedName (tokenInfo *const nameToken)
788834
vString *qualifiedName = vStringNew ();
789835
tokenInfo *token = newToken ();
790836

837+
unsigned long lineNumber = nameToken->lineNumber;
838+
MIOPos filePosition = nameToken->filePosition;
839+
840+
useTokenAsPartOfAnotherTag (nameToken);
791841
while (nameToken->type == TOKEN_IDENTIFIER ||
792842
nameToken->type == '.')
793843
{
794844
vStringCat (qualifiedName, nameToken->string);
795845
copyToken (token, nameToken);
796846

797847
readToken (nameToken);
848+
useTokenAsPartOfAnotherTag (nameToken);
798849
}
799850
/* put the last, non-matching, token back */
800851
ungetToken (nameToken);
@@ -805,6 +856,13 @@ static void readQualifiedName (tokenInfo *const nameToken)
805856

806857
deleteToken (token);
807858
vStringDelete (qualifiedName);
859+
860+
tagEntryInfo e;
861+
initRefTagEntry(&e, vStringValue (nameToken->string),
862+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
863+
e.lineNumber = lineNumber;
864+
e.filePosition = filePosition;
865+
nameToken->reftag = makeTagEntry (&e);
808866
}
809867
}
810868

@@ -1024,6 +1082,7 @@ static void parseArglist (tokenInfo *const token, const int kind,
10241082
struct typedParam *parameter;
10251083

10261084
parameterName = newToken ();
1085+
useTokenAsPartOfAnotherTag (token);
10271086
copyToken (parameterName, token);
10281087
parameterType = parseParamTypeAnnotation (token, arglist);
10291088

@@ -1490,7 +1549,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind)
14901549

14911550
do
14921551
{
1493-
const tokenInfo *const nameToken = nameTokens[i];
1552+
tokenInfo *const nameToken = nameTokens[i];
14941553
vString **type = &(nameTypes[i++]);
14951554

14961555
readToken (token);

0 commit comments

Comments
 (0)