Skip to content

Commit 1580ec2

Browse files
masatakemasatake
committed
Python: extract reference tags
Signed-off-by: Masatake YAMATO <[email protected]>
1 parent 293f11e commit 1580ec2

File tree

6 files changed

+115
-9
lines changed

6 files changed

+115
-9
lines changed

‎Tmain/json-output-format.d/stdout-expected.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "s,section", "pattern": "sections"}
7676
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Man", "path": "t,title", "pattern": "titles"}
7777
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "I,namespace", "pattern": "name referring a module defined in other file"}
78-
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "name referring a class/variable/function/module defined in other module"}
78+
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "Y,unknown", "pattern": "unknwon name"}
7979
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "c,class", "pattern": "classes"}
8080
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "f,function", "pattern": "functions"}
8181
{"_type": "ptag", "name": "TAG_KIND_DESCRIPTION", "parserName": "Python", "path": "i,module", "pattern": "modules"}

‎Tmain/list-roles.d/stdout-expected.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ Protobuf D/protodef imported on imported
9191
Protobuf m/message extension on extending the message
9292
Python Y/unknown imported on imported from the other module
9393
Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
94+
Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow
9495
Python i/module imported on imported modules
9596
Python i/module indirectlyImported on module imported in alternative name
9697
Python i/module namespace on namespace from where classes/variables/functions are imported
@@ -224,6 +225,7 @@ Protobuf D/protodef imported on imported
224225
Protobuf m/message extension on extending the message
225226
Python Y/unknown imported on imported from the other module
226227
Python Y/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
228+
Python Y/unknown ref off (EXPERIMENTAL)referenced anyhow
227229
Python i/module imported on imported modules
228230
Python i/module indirectlyImported on module imported in alternative name
229231
Python i/module namespace on namespace from where classes/variables/functions are imported

‎Units/parser-python.r/reftag.d/args.ctags‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
--sort=no
2+
--extras=+r
3+
--fields=+r
4+
--kinds-Python=*
5+
--roles-Python.{unknown}=+{ref}

‎Units/parser-python.r/reftag.d/expected.tags‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
a.b input.py /^import a.b$/;" i roles:imported
2+
object input.py /^class X(object):$/;" Y roles:ref
3+
X input.py /^class X(object):$/;" c roles:def
4+
__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def
5+
self input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def
6+
n input.py /^ def __init__(self, n):$/;" z member:X.__init__ file: roles:def
7+
self input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
8+
n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
9+
n input.py /^ self.n = n$/;" Y member:X.__init__ roles:ref
10+
val input.py /^ def val(self):$/;" m class:X roles:def
11+
self input.py /^ def val(self):$/;" z member:X.val file: roles:def
12+
self input.py /^ return self.n$/;" Y member:X.val roles:ref
13+
n input.py /^ return self.n$/;" Y member:X.val roles:ref
14+
one input.py /^def one():$/;" f roles:def
15+
X input.py /^ return X(1)$/;" Y function:one roles:ref
16+
two input.py /^def two():$/;" f roles:def
17+
X input.py /^ return X(2)$/;" Y function:two roles:ref
18+
print input.py /^print (one().val() + two().val())$/;" Y roles:ref
19+
one input.py /^print (one().val() + two().val())$/;" Y roles:ref
20+
val input.py /^print (one().val() + two().val())$/;" Y roles:ref
21+
two input.py /^print (one().val() + two().val())$/;" Y roles:ref
22+
val input.py /^print (one().val() + two().val())$/;" Y roles:ref

‎Units/parser-python.r/reftag.d/input.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import a.b
2+
3+
class X(object):
4+
def __init__(self, n):
5+
self.n = n
6+
7+
def val(self):
8+
return self.n
9+
10+
def one():
11+
return X(1)
12+
13+
def two():
14+
return X(2)
15+
16+
print (one().val() + two().val())
17+
18+

‎parsers/python.c‎

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ typedef enum {
9696
} pythonModuleRole;
9797

9898
typedef enum {
99+
PYTHON_UNKNOWN_REFERENCED,
99100
PYTHON_UNKNOWN_IMPORTED,
100101
PYTHON_UNKNOWN_INDIRECTLY_IMPORTED,
101102
} pythonUnknownRole;
@@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = {
128129
};
129130

130131
static roleDefinition PythonUnknownRoles [] = {
132+
{ false,"ref", "(EXPERIMENTAL)referenced anyhow" },
131133
{ true, "imported", "imported from the other module" },
132134
{ true, "indirectlyImported",
133135
"classes/variables/functions/modules imported in alternative name" },
@@ -141,7 +143,7 @@ static kindDefinition PythonKinds[COUNT_KIND] = {
141143
{true, 'I', "namespace", "name referring a module defined in other file"},
142144
{true, 'i', "module", "modules",
143145
.referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)},
144-
{true, 'Y', "unknown", "name referring a class/variable/function/module defined in other module",
146+
{true, 'Y', "unknown", "unknwon name",
145147
.referenceOnly = false, ATTACH_ROLES(PythonUnknownRoles)},
146148
{false, 'z', "parameter", "function parameters" },
147149
{false, 'l', "local", "local variables" },
@@ -201,6 +203,7 @@ typedef struct {
201203
int indent;
202204
unsigned long lineNumber;
203205
MIOPos filePosition;
206+
int reftag;
204207
} tokenInfo;
205208

206209
struct pythonNestingLevelUserData {
@@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident,
244247
return ACCESS_PROTECTED;
245248
}
246249

247-
static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token,
250+
static void useTokenAsPartOfAnotherTag (tokenInfo *const token)
251+
{
252+
if (token->reftag == CORK_NIL)
253+
return;
254+
255+
markCorkEntryAsPlaceholder (token->reftag, true);
256+
token->reftag = CORK_NIL;
257+
}
258+
259+
static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token,
248260
const pythonKind kind)
249261
{
250262
accessType access;
251263
int parentKind = -1;
252264
NestingLevel *nl;
253265

266+
useTokenAsPartOfAnotherTag(token);
254267
initTagEntry (e, vStringValue (token->string), kind);
255268

256269
updateTagLine (e, token->lineNumber, token->filePosition);
@@ -283,7 +296,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token
283296
e->isFileScope = true;
284297
}
285298

286-
static int makeClassTag (const tokenInfo *const token,
299+
static int makeClassTag (tokenInfo *const token,
287300
const vString *const inheritance,
288301
const vString *const decorators)
289302
{
@@ -306,7 +319,7 @@ static int makeClassTag (const tokenInfo *const token,
306319
return CORK_NIL;
307320
}
308321

309-
static int makeFunctionTag (const tokenInfo *const token,
322+
static int makeFunctionTag (tokenInfo *const token,
310323
const vString *const arglist,
311324
const vString *const decorators)
312325
{
@@ -330,7 +343,7 @@ static int makeFunctionTag (const tokenInfo *const token,
330343
return CORK_NIL;
331344
}
332345

333-
static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind)
346+
static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind)
334347
{
335348
if (PythonKinds[kind].enabled)
336349
{
@@ -343,7 +356,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k
343356
return CORK_NIL;
344357
}
345358

346-
static int makeSimplePythonRefTag (const tokenInfo *const token,
359+
static int makeSimplePythonRefTag (tokenInfo *const token,
347360
const vString *const altName,
348361
pythonKind const kind,
349362
int roleIndex, xtagType xtag)
@@ -353,6 +366,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token,
353366
{
354367
tagEntryInfo e;
355368

369+
useTokenAsPartOfAnotherTag(token);
356370
initRefTagEntry (&e, vStringValue (altName ? altName : token->string),
357371
kind, roleIndex);
358372

@@ -391,6 +405,7 @@ static void clearPoolToken (void *data)
391405
token->lineNumber = getInputLineNumber ();
392406
token->filePosition = getInputFilePosition ();
393407
vStringClear (token->string);
408+
token->reftag = CORK_NIL;
394409
}
395410

396411
static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
@@ -401,6 +416,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
401416
dest->keyword = src->keyword;
402417
dest->indent = src->indent;
403418
vStringCopy(dest->string, src->string);
419+
dest->reftag = src->reftag;
404420
}
405421

406422
/* Skip a single or double quoted string. */
@@ -477,7 +493,7 @@ static void ungetToken (tokenInfo *const token)
477493
copyToken (NextToken, token);
478494
}
479495

480-
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
496+
static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces)
481497
{
482498
int c;
483499
int n;
@@ -692,6 +708,36 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
692708
}
693709
}
694710

711+
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
712+
{
713+
readTokenFullNoRefTag (token, inclWhitespaces);
714+
715+
if (token->type == TOKEN_IDENTIFIER
716+
/* Don't make a ref tag for a number. */
717+
&& (vStringLength(token->string) > 0 &&
718+
!isdigit((unsigned char)vStringChar(token->string, 0)))
719+
&& PythonKinds[K_UNKNOWN].enabled
720+
&& PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled)
721+
{
722+
const bool in_subparser = (Lang_python != getInputLanguage());
723+
if (in_subparser)
724+
pushLanguage(Lang_python);
725+
726+
tagEntryInfo e;
727+
initRefTagEntry(&e, vStringValue (token->string),
728+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
729+
e.lineNumber = token->lineNumber;
730+
e.filePosition = token->filePosition;
731+
NestingLevel *nl = nestingLevelsGetCurrent (PythonNestingLevels);
732+
if (nl)
733+
e.extensionFields.scopeIndex = nl->corkIndex;
734+
token->reftag = makeTagEntry (&e);
735+
736+
if (in_subparser)
737+
popLanguage();
738+
}
739+
}
740+
695741
static void readToken (tokenInfo *const token)
696742
{
697743
readTokenFull (token, false);
@@ -786,13 +832,18 @@ static void readQualifiedName (tokenInfo *const nameToken)
786832
vString *qualifiedName = vStringNew ();
787833
tokenInfo *token = newToken ();
788834

835+
unsigned long lineNumber = nameToken->lineNumber;
836+
MIOPos filePosition = nameToken->filePosition;
837+
838+
useTokenAsPartOfAnotherTag (nameToken);
789839
while (nameToken->type == TOKEN_IDENTIFIER ||
790840
nameToken->type == '.')
791841
{
792842
vStringCat (qualifiedName, nameToken->string);
793843
copyToken (token, nameToken);
794844

795845
readToken (nameToken);
846+
useTokenAsPartOfAnotherTag (nameToken);
796847
}
797848
/* put the last, non-matching, token back */
798849
ungetToken (nameToken);
@@ -803,6 +854,13 @@ static void readQualifiedName (tokenInfo *const nameToken)
803854

804855
deleteToken (token);
805856
vStringDelete (qualifiedName);
857+
858+
tagEntryInfo e;
859+
initRefTagEntry(&e, vStringValue (nameToken->string),
860+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
861+
e.lineNumber = lineNumber;
862+
e.filePosition = filePosition;
863+
nameToken->reftag = makeTagEntry (&e);
806864
}
807865
}
808866

@@ -1022,6 +1080,7 @@ static void parseArglist (tokenInfo *const token, const int kind,
10221080
struct typedParam *parameter;
10231081

10241082
parameterName = newToken ();
1083+
useTokenAsPartOfAnotherTag (token);
10251084
copyToken (parameterName, token);
10261085
parameterType = parseParamTypeAnnotation (token, arglist);
10271086

@@ -1488,7 +1547,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind)
14881547

14891548
do
14901549
{
1491-
const tokenInfo *const nameToken = nameTokens[i];
1550+
tokenInfo *const nameToken = nameTokens[i];
14921551
vString **type = &(nameTypes[i++]);
14931552

14941553
readToken (token);

0 commit comments

Comments
 (0)