2020import tokenize
2121from abc import ABCMeta
2222from ast import Module , expr , AST
23- from typing import Callable , Dict , Iterable , Iterator , List , Optional , Tuple , Union , cast , Any , TYPE_CHECKING
24-
25- import astroid
26-
23+ from functools import lru_cache
24+ from typing import (
25+ Callable ,
26+ Dict ,
27+ Iterable ,
28+ Iterator ,
29+ List ,
30+ Optional ,
31+ Tuple ,
32+ Union ,
33+ cast ,
34+ Any ,
35+ TYPE_CHECKING ,
36+ Type ,
37+ )
2738
2839if TYPE_CHECKING : # pragma: no cover
2940 from .astroid_compat import NodeNG
@@ -67,13 +78,6 @@ def __str__(self):
6778 return token_repr (self .type , self .string )
6879
6980
70- if sys .version_info >= (3 , 6 ):
71- AstConstant = ast .Constant
72- else :
73- class AstConstant :
74- value = object ()
75-
76-
7781def match_token (token , tok_type , tok_str = None ):
7882 # type: (Token, int, Optional[str]) -> bool
7983 """Returns true if token is of the given type and, if a string is given, has that string."""
@@ -91,22 +95,13 @@ def expect_token(token, tok_type, tok_str=None):
9195 token_repr (tok_type , tok_str ), str (token ),
9296 token .start [0 ], token .start [1 ] + 1 ))
9397
94- # These were previously defined in tokenize.py and distinguishable by being greater than
95- # token.N_TOKEN. As of python3.7, they are in token.py, and we check for them explicitly.
96- if sys .version_info >= (3 , 7 ):
97- def is_non_coding_token (token_type ):
98- # type: (int) -> bool
99- """
100- These are considered non-coding tokens, as they don't affect the syntax tree.
101- """
102- return token_type in (token .NL , token .COMMENT , token .ENCODING )
103- else :
104- def is_non_coding_token (token_type ):
105- # type: (int) -> bool
106- """
107- These are considered non-coding tokens, as they don't affect the syntax tree.
108- """
109- return token_type >= token .N_TOKENS
98+
99+ def is_non_coding_token (token_type ):
100+ # type: (int) -> bool
101+ """
102+ These are considered non-coding tokens, as they don't affect the syntax tree.
103+ """
104+ return token_type in (token .NL , token .COMMENT , token .ENCODING )
110105
111106
112107def generate_tokens (text ):
@@ -201,10 +196,19 @@ def is_expr_stmt(node):
201196 return node .__class__ .__name__ == 'Expr'
202197
203198
199+
200+ CONSTANT_CLASSES : Tuple [Type , ...] = (ast .Constant ,)
201+ try :
202+ from astroid import Const
203+ CONSTANT_CLASSES += (Const ,)
204+ except ImportError : # pragma: no cover
205+ # astroid is not available
206+ pass
207+
204208def is_constant (node ):
205209 # type: (AstNode) -> bool
206210 """Returns whether node is a Constant node."""
207- return isinstance (node , ( ast . Constant , astroid . Const ) )
211+ return isinstance (node , CONSTANT_CLASSES )
208212
209213
210214def is_ellipsis (node ):
@@ -421,72 +425,61 @@ def last_stmt(node):
421425 return node
422426
423427
424- if sys .version_info [:2 ] >= (3 , 8 ):
425- from functools import lru_cache
426428
427- @lru_cache (maxsize = None )
428- def fstring_positions_work ():
429- # type: () -> bool
430- """
431- The positions attached to nodes inside f-string FormattedValues have some bugs
432- that were fixed in Python 3.9.7 in https://github.com/python/cpython/pull/27729.
433- This checks for those bugs more concretely without relying on the Python version.
434- Specifically this checks:
435- - Values with a format spec or conversion
436- - Repeated (i.e. identical-looking) expressions
437- - f-strings implicitly concatenated over multiple lines.
438- - Multiline, triple-quoted f-strings.
439- """
440- source = """(
441- f"a {b}{b} c {d!r} e {f:g} h {i:{j}} k {l:{m:n}}"
442- f"a {b}{b} c {d!r} e {f:g} h {i:{j}} k {l:{m:n}}"
443- f"{x + y + z} {x} {y} {z} {z} {z!a} {z:z}"
444- f'''
445- {s} {t}
446- {u} {v}
447- '''
448- )"""
449- tree = ast .parse (source )
450- name_nodes = [node for node in ast .walk (tree ) if isinstance (node , ast .Name )]
451- name_positions = [(node .lineno , node .col_offset ) for node in name_nodes ]
452- positions_are_unique = len (set (name_positions )) == len (name_positions )
453- correct_source_segments = all (
454- ast .get_source_segment (source , node ) == node .id
455- for node in name_nodes
456- )
457- return positions_are_unique and correct_source_segments
429+ @lru_cache (maxsize = None )
430+ def fstring_positions_work ():
431+ # type: () -> bool
432+ """
433+ The positions attached to nodes inside f-string FormattedValues have some bugs
434+ that were fixed in Python 3.9.7 in https://github.com/python/cpython/pull/27729.
435+ This checks for those bugs more concretely without relying on the Python version.
436+ Specifically this checks:
437+ - Values with a format spec or conversion
438+ - Repeated (i.e. identical-looking) expressions
439+ - f-strings implicitly concatenated over multiple lines.
440+ - Multiline, triple-quoted f-strings.
441+ """
442+ source = """(
443+ f"a {b}{b} c {d!r} e {f:g} h {i:{j}} k {l:{m:n}}"
444+ f"a {b}{b} c {d!r} e {f:g} h {i:{j}} k {l:{m:n}}"
445+ f"{x + y + z} {x} {y} {z} {z} {z!a} {z:z}"
446+ f'''
447+ {s} {t}
448+ {u} {v}
449+ '''
450+ )"""
451+ tree = ast .parse (source )
452+ name_nodes = [node for node in ast .walk (tree ) if isinstance (node , ast .Name )]
453+ name_positions = [(node .lineno , node .col_offset ) for node in name_nodes ]
454+ positions_are_unique = len (set (name_positions )) == len (name_positions )
455+ correct_source_segments = all (
456+ ast .get_source_segment (source , node ) == node .id
457+ for node in name_nodes
458+ )
459+ return positions_are_unique and correct_source_segments
458460
459- def annotate_fstring_nodes (tree ):
460- # type: (ast.AST) -> None
461- """
462- Add a special attribute `_broken_positions` to nodes inside f-strings
463- if the lineno/col_offset cannot be trusted.
464- """
465- if sys .version_info >= (3 , 12 ):
466- # f-strings were weirdly implemented until https://peps.python.org/pep-0701/
467- # In Python 3.12, inner nodes have sensible positions.
468- return
469- for joinedstr in walk (tree , include_joined_str = True ):
470- if not isinstance (joinedstr , ast .JoinedStr ):
471- continue
472- for part in joinedstr .values :
473- # The ast positions of the FormattedValues/Constant nodes span the full f-string, which is weird.
474- setattr (part , '_broken_positions' , True ) # use setattr for mypy
475-
476- if isinstance (part , ast .FormattedValue ):
477- if not fstring_positions_work ():
478- for child in walk (part .value ):
479- setattr (child , '_broken_positions' , True )
480-
481- if part .format_spec : # this is another JoinedStr
482- # Again, the standard positions span the full f-string.
483- setattr (part .format_spec , '_broken_positions' , True )
484-
485- else :
486- def fstring_positions_work ():
487- # type: () -> bool
488- return False
489-
490- def annotate_fstring_nodes (_tree ):
491- # type: (ast.AST) -> None
492- pass
461+ def annotate_fstring_nodes (tree ):
462+ # type: (ast.AST) -> None
463+ """
464+ Add a special attribute `_broken_positions` to nodes inside f-strings
465+ if the lineno/col_offset cannot be trusted.
466+ """
467+ if sys .version_info >= (3 , 12 ):
468+ # f-strings were weirdly implemented until https://peps.python.org/pep-0701/
469+ # In Python 3.12, inner nodes have sensible positions.
470+ return
471+ for joinedstr in walk (tree , include_joined_str = True ):
472+ if not isinstance (joinedstr , ast .JoinedStr ):
473+ continue
474+ for part in joinedstr .values :
475+ # The ast positions of the FormattedValues/Constant nodes span the full f-string, which is weird.
476+ setattr (part , '_broken_positions' , True ) # use setattr for mypy
477+
478+ if isinstance (part , ast .FormattedValue ):
479+ if not fstring_positions_work ():
480+ for child in walk (part .value ):
481+ setattr (child , '_broken_positions' , True )
482+
483+ if part .format_spec : # this is another JoinedStr
484+ # Again, the standard positions span the full f-string.
485+ setattr (part .format_spec , '_broken_positions' , True )
0 commit comments