Skip to content

Commit 8f52d02

Browse files
masatakemasatake
committed
Python: extract reference tags
Signed-off-by: Masatake YAMATO <[email protected]>
1 parent aacb0eb commit 8f52d02

File tree

5 files changed

+104
-7
lines changed

5 files changed

+104
-7
lines changed

‎Tmain/list-roles.d/stdout-expected.txt

+2
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ Python i/module indirectlyImported on module imported in a
8787
Python i/module namespace on namespace from where classes/variables/functions are imported
8888
Python x/unknown imported on imported from the other module
8989
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
90+
Python x/unknown ref off referenced anyhow
9091
R l/library library on library attached by library function
9192
R l/library require on library attached by require function
9293
R s/source source on source loaded by source fucntion
@@ -208,6 +209,7 @@ Python i/module indirectlyImported on module imported in a
208209
Python i/module namespace on namespace from where classes/variables/functions are imported
209210
Python x/unknown imported on imported from the other module
210211
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
212+
Python x/unknown ref off referenced anyhow
211213
R l/library library on library attached by library function
212214
R l/library require on library attached by require function
213215
R s/source source on source loaded by source fucntion

‎Units/parser-python.r/reftag.d/args.ctags

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
--sort=no
2+
--extras=+r
3+
--fields=+r
4+
--roles-Python.{unknown}=+{ref}

‎Units/parser-python.r/reftag.d/expected.tags

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
a.b input.py /^import a.b$/;" i roles:imported
2+
object input.py /^class X(object):$/;" x roles:ref
3+
X input.py /^class X(object):$/;" c roles:def
4+
self input.py /^ def __init__(self, n):$/;" x roles:ref
5+
n input.py /^ def __init__(self, n):$/;" x roles:ref
6+
__init__ input.py /^ def __init__(self, n):$/;" m class:X roles:def
7+
self input.py /^ self.n = n$/;" x roles:ref
8+
n input.py /^ self.n = n$/;" x roles:ref
9+
n input.py /^ self.n = n$/;" x roles:ref
10+
self input.py /^ def val(self):$/;" x roles:ref
11+
val input.py /^ def val(self):$/;" m class:X roles:def
12+
self input.py /^ return self.n$/;" x roles:ref
13+
n input.py /^ return self.n$/;" x roles:ref
14+
one input.py /^def one():$/;" f roles:def
15+
X input.py /^ return X(1)$/;" x roles:ref
16+
two input.py /^def two():$/;" f roles:def
17+
X input.py /^ return X(2)$/;" x roles:ref
18+
print input.py /^print (one().val() + two().val())$/;" x roles:ref
19+
one input.py /^print (one().val() + two().val())$/;" x roles:ref
20+
val input.py /^print (one().val() + two().val())$/;" x roles:ref
21+
two input.py /^print (one().val() + two().val())$/;" x roles:ref
22+
val input.py /^print (one().val() + two().val())$/;" x roles:ref

‎Units/parser-python.r/reftag.d/input.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import a.b
2+
3+
class X(object):
4+
def __init__(self, n):
5+
self.n = n
6+
7+
def val(self):
8+
return self.n
9+
10+
def one():
11+
return X(1)
12+
13+
def two():
14+
return X(2)
15+
16+
print (one().val() + two().val())
17+
18+

‎parsers/python.c

+58-7
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ typedef enum {
9696
} pythonModuleRole;
9797

9898
typedef enum {
99+
PYTHON_UNKNOWN_REFERENCED,
99100
PYTHON_UNKNOWN_IMPORTED,
100101
PYTHON_UNKNOWN_INDIRECTLY_IMPORTED,
101102
} pythonUnknownRole;
@@ -128,6 +129,7 @@ static roleDefinition PythonModuleRoles [] = {
128129
};
129130

130131
static roleDefinition PythonUnknownRoles [] = {
132+
{ false,"ref", "referenced anyhow" },
131133
{ true, "imported", "imported from the other module" },
132134
{ true, "indirectlyImported",
133135
"classes/variables/functions/modules imported in alternative name" },
@@ -201,6 +203,7 @@ typedef struct {
201203
int indent;
202204
unsigned long lineNumber;
203205
MIOPos filePosition;
206+
int reftag;
204207
} tokenInfo;
205208

206209
struct pythonNestingLevelUserData {
@@ -244,13 +247,23 @@ static accessType accessFromIdentifier (const vString *const ident,
244247
return ACCESS_PROTECTED;
245248
}
246249

247-
static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token,
250+
static void useTokenAsPartOfTag (tokenInfo *const token)
251+
{
252+
if (token->reftag == CORK_NIL)
253+
return;
254+
255+
markCorkEntryPlaceholder (token->reftag, true);
256+
token->reftag = CORK_NIL;
257+
}
258+
259+
static void initPythonEntry (tagEntryInfo *const e, tokenInfo *const token,
248260
const pythonKind kind)
249261
{
250262
accessType access;
251263
int parentKind = -1;
252264
NestingLevel *nl;
253265

266+
useTokenAsPartOfTag(token);
254267
initTagEntry (e, vStringValue (token->string), kind);
255268

256269
e->lineNumber = token->lineNumber;
@@ -284,7 +297,7 @@ static void initPythonEntry (tagEntryInfo *const e, const tokenInfo *const token
284297
e->isFileScope = true;
285298
}
286299

287-
static int makeClassTag (const tokenInfo *const token,
300+
static int makeClassTag (tokenInfo *const token,
288301
const vString *const inheritance,
289302
const vString *const decorators)
290303
{
@@ -307,7 +320,7 @@ static int makeClassTag (const tokenInfo *const token,
307320
return CORK_NIL;
308321
}
309322

310-
static int makeFunctionTag (const tokenInfo *const token,
323+
static int makeFunctionTag (tokenInfo *const token,
311324
const vString *const arglist,
312325
const vString *const decorators)
313326
{
@@ -331,7 +344,7 @@ static int makeFunctionTag (const tokenInfo *const token,
331344
return CORK_NIL;
332345
}
333346

334-
static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const kind)
347+
static int makeSimplePythonTag (tokenInfo *const token, pythonKind const kind)
335348
{
336349
if (PythonKinds[kind].enabled)
337350
{
@@ -344,7 +357,7 @@ static int makeSimplePythonTag (const tokenInfo *const token, pythonKind const k
344357
return CORK_NIL;
345358
}
346359

347-
static int makeSimplePythonRefTag (const tokenInfo *const token,
360+
static int makeSimplePythonRefTag (tokenInfo *const token,
348361
const vString *const altName,
349362
pythonKind const kind,
350363
int roleIndex, xtagType xtag)
@@ -354,6 +367,7 @@ static int makeSimplePythonRefTag (const tokenInfo *const token,
354367
{
355368
tagEntryInfo e;
356369

370+
useTokenAsPartOfTag(token);
357371
initRefTagEntry (&e, vStringValue (altName ? altName : token->string),
358372
kind, roleIndex);
359373

@@ -393,6 +407,7 @@ static void clearPoolToken (void *data)
393407
token->lineNumber = getInputLineNumber ();
394408
token->filePosition = getInputFilePosition ();
395409
vStringClear (token->string);
410+
token->reftag = CORK_NIL;
396411
}
397412

398413
static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
@@ -403,6 +418,7 @@ static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
403418
dest->keyword = src->keyword;
404419
dest->indent = src->indent;
405420
vStringCopy(dest->string, src->string);
421+
dest->reftag = src->reftag;
406422
}
407423

408424
/* Skip a single or double quoted string. */
@@ -479,7 +495,7 @@ static void ungetToken (tokenInfo *const token)
479495
copyToken (NextToken, token);
480496
}
481497

482-
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
498+
static void readTokenFullNoRefTag (tokenInfo *const token, bool inclWhitespaces)
483499
{
484500
int c;
485501
int n;
@@ -694,6 +710,29 @@ static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
694710
}
695711
}
696712

713+
static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
714+
{
715+
readTokenFullNoRefTag (token, inclWhitespaces);
716+
717+
if (token->type == TOKEN_IDENTIFIER
718+
/* Don't make a ref tag for a number. */
719+
&& (vStringLength(token->string) > 0 &&
720+
!isdigit((unsigned char)vStringChar(token->string, 0)))
721+
&& PythonKinds[K_UNKNOWN].enabled
722+
&& PythonUnknownRoles[PYTHON_UNKNOWN_REFERENCED].enabled)
723+
{
724+
const bool in_subparser = (Lang_python != getInputLanguage());
725+
if (in_subparser)
726+
pushLanguage(Lang_python);
727+
728+
token->reftag = makeSimpleRefTag (token->string,
729+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
730+
731+
if (in_subparser)
732+
popLanguage();
733+
}
734+
}
735+
697736
static void readToken (tokenInfo *const token)
698737
{
699738
readTokenFull (token, false);
@@ -788,13 +827,18 @@ static void readQualifiedName (tokenInfo *const nameToken)
788827
vString *qualifiedName = vStringNew ();
789828
tokenInfo *token = newToken ();
790829

830+
unsigned long lineNumber = nameToken->lineNumber;
831+
MIOPos filePosition = nameToken->filePosition;
832+
useTokenAsPartOfTag(token);
833+
791834
while (nameToken->type == TOKEN_IDENTIFIER ||
792835
nameToken->type == '.')
793836
{
794837
vStringCat (qualifiedName, nameToken->string);
795838
copyToken (token, nameToken);
796839

797840
readToken (nameToken);
841+
useTokenAsPartOfTag (nameToken);
798842
}
799843
/* put the last, non-matching, token back */
800844
ungetToken (nameToken);
@@ -805,6 +849,13 @@ static void readQualifiedName (tokenInfo *const nameToken)
805849

806850
deleteToken (token);
807851
vStringDelete (qualifiedName);
852+
853+
tagEntryInfo e;
854+
initRefTagEntry(&e, vStringValue (nameToken->string),
855+
K_UNKNOWN, PYTHON_UNKNOWN_REFERENCED);
856+
e.lineNumber = lineNumber;
857+
e.filePosition = filePosition;
858+
nameToken->reftag = makeTagEntry (&e);
808859
}
809860
}
810861

@@ -1490,7 +1541,7 @@ static bool parseVariable (tokenInfo *const token, const pythonKind kind)
14901541

14911542
do
14921543
{
1493-
const tokenInfo *const nameToken = nameTokens[i];
1544+
tokenInfo *const nameToken = nameTokens[i];
14941545
vString **type = &(nameTypes[i++]);
14951546

14961547
readToken (token);

0 commit comments

Comments
 (0)