Skip to content

Use AST token constants from token module. #14701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use AST token constants from token module. #14701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 23 additions & 22 deletions IPython/core/completer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@
import re
import string
import sys
import token
import tokenize
import time
import unicodedata
Expand Down Expand Up @@ -1175,16 +1176,16 @@ def _attr_matches(
name_turn = True

parts = []
for token in rev_tokens:
if token.type in skip_over:
for tok in rev_tokens:
if tok.type in skip_over:
continue
if token.type == tokenize.NAME and name_turn:
parts.append(token.string)
if tok.type == tokenize.NAME and name_turn:
parts.append(tok.string)
name_turn = False
elif (
token.type == tokenize.OP and token.string == "." and not name_turn
tok.type == token.OP and tok.string == "." and not name_turn
):
parts.append(token.string)
parts.append(tok.string)
name_turn = True
else:
# short-circuit if not empty nor name token
Expand Down Expand Up @@ -1298,23 +1299,23 @@ def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]:
return None
tokens = _parse_tokens(prefix)
rev_tokens = reversed(tokens)
skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
skip_over = {token.ENDMARKER, token.NEWLINE}
number = None
for token in rev_tokens:
if token.type in skip_over:
for tok in rev_tokens:
if tok.type in skip_over:
continue
if number is None:
if token.type == tokenize.NUMBER:
number = token.string
if tok.type == token.NUMBER:
number = tok.string
continue
else:
# we did not match a number
return None
if token.type == tokenize.OP:
if token.string == ",":
if tok.type == tokenize.OP:
if tok.string == ",":
break
if token.string in {"+", "-"}:
number = token.string + number
if tok.string in {"+", "-"}:
number = tok.string + number
else:
return None
return number
Expand Down Expand Up @@ -2491,10 +2492,10 @@ def python_func_kw_matches(self, text):
iterTokens = reversed(tokens)
openPar = 0

for token in iterTokens:
if token == ')':
for tok in iterTokens:
if tok == ')':
openPar -= 1
elif token == '(':
elif tok == '(':
openPar += 1
if openPar > 0:
# found the last unclosed parenthesis
Expand All @@ -2520,10 +2521,10 @@ def python_func_kw_matches(self, text):
# them again
usedNamedArgs = set()
par_level = -1
for token, next_token in zip(tokens, tokens[1:]):
if token == '(':
for tok, next_token in zip(tokens, tokens[1:]):
if tok == '(':
par_level += 1
elif token == ')':
elif tok == ')':
par_level -= 1

if par_level != 0:
Expand All @@ -2532,7 +2533,7 @@ def python_func_kw_matches(self, text):
if next_token != '=':
continue

usedNamedArgs.add(token)
usedNamedArgs.add(tok)

argMatches = []
try:
Expand Down
7 changes: 4 additions & 3 deletions IPython/core/displayhook.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import builtins as builtin_mod
import sys
import io as _io
import token
import tokenize

from traitlets.config.configurable import Configurable
Expand Down Expand Up @@ -100,10 +101,10 @@ def semicolon_at_end_of_expression(expression):
sio = _io.StringIO(expression)
tokens = list(tokenize.generate_tokens(sio.readline))

for token in reversed(tokens):
if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT):
for tok in reversed(tokens):
if tok[0] in (token.ENDMARKER, token.NL, token.NEWLINE, token.COMMENT):
continue
if (token[0] == tokenize.OP) and (token[1] == ';'):
if (tok[0] == token.OP) and (tok[1] == ';'):
return True
else:
return False
Expand Down
48 changes: 23 additions & 25 deletions IPython/core/inputtransformer2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from codeop import CommandCompiler, Compile
import re
import sys
import token
import tokenize
from typing import List, Tuple, Optional, Any
import warnings
Expand Down Expand Up @@ -170,7 +171,7 @@ def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
if (assign_ix is not None) \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix+1].string == '%') \
and (line[assign_ix+2].type == tokenize.NAME):
and (line[assign_ix+2].type == token.NAME):

This statement contains four continued line pieces.
Assembling these pieces into a single line would give::
Expand Down Expand Up @@ -248,7 +249,7 @@ def find(cls, tokens_by_line):
if (assign_ix is not None) \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix+1].string == '%') \
and (line[assign_ix+2].type == tokenize.NAME):
and (line[assign_ix+2].type == token.NAME):
return cls(line[assign_ix+1].start)

def transform(self, lines: List[str]):
Expand Down Expand Up @@ -278,10 +279,10 @@ def find_pre_312(cls, tokens_by_line):
if (assign_ix is not None) \
and not line[assign_ix].line.strip().startswith('=') \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
and (line[assign_ix + 1].type == token.ERRORTOKEN):
ix = assign_ix + 1

while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
while ix < len(line) and line[ix].type == token.ERRORTOKEN:
if line[ix].string == '!':
return cls(line[ix].start)
elif not line[ix].string.isspace():
Expand All @@ -296,7 +297,7 @@ def find_post_312(cls, tokens_by_line):
(assign_ix is not None)
and not line[assign_ix].line.strip().startswith("=")
and (len(line) >= assign_ix + 2)
and (line[assign_ix + 1].type == tokenize.OP)
and (line[assign_ix + 1].type == token.OP)
and (line[assign_ix + 1].string == "!")
):
return cls(line[assign_ix + 1].start)
Expand Down Expand Up @@ -421,7 +422,7 @@ def find(cls, tokens_by_line):
continue
ix = 0
ll = len(line)
while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
while ll > ix and line[ix].type in {token.INDENT, token.DEDENT}:
ix += 1
if ix >= ll:
continue
Expand Down Expand Up @@ -485,7 +486,7 @@ def find(cls, tokens_by_line):
if len(line) > 2 and line[-2].string == '?':
# Find the first token that's not INDENT/DEDENT
ix = 0
while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
while line[ix].type in {token.INDENT, token.DEDENT}:
ix += 1
return cls(line[ix].start, line[-2].start)

Expand Down Expand Up @@ -525,8 +526,6 @@ def make_tokens_by_line(lines:List[str]):
# We want to group the former case together but split the latter, so we
# track parentheses level, similar to the internals of tokenize.

# reexported from token on 3.7+
NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
tokens_by_line: List[List[Any]] = [[]]
if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
warnings.warn(
Expand All @@ -535,16 +534,15 @@ def make_tokens_by_line(lines:List[str]):
)
parenlev = 0
try:
for token in tokenutil.generate_tokens_catch_errors(
for tok in tokenutil.generate_tokens_catch_errors(
iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
):
tokens_by_line[-1].append(token)
if (token.type == NEWLINE) \
or ((token.type == NL) and (parenlev <= 0)):
tokens_by_line[-1].append(tok)
if (tok.type == NEWLINE) or ((tok.type == NL) and (parenlev <= 0)):
tokens_by_line.append([])
elif token.string in {'(', '[', '{'}:
elif tok.string in {'(', '[', '{'}:
parenlev += 1
elif token.string in {')', ']', '}'}:
elif tok.string in {')', ']', '}'}:
if parenlev > 0:
parenlev -= 1
except tokenize.TokenError:
Expand All @@ -562,10 +560,10 @@ def make_tokens_by_line(lines:List[str]):
def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
"""Check if the depth of brackets in the list of tokens drops below 0"""
parenlev = 0
for token in tokens:
if token.string in {"(", "[", "{"}:
for tok in tokens:
if tok.string in {"(", "[", "{"}:
parenlev += 1
elif token.string in {")", "]", "}"}:
elif tok.string in {")", "]", "}"}:
parenlev -= 1
if parenlev < 0:
return True
Expand Down Expand Up @@ -732,19 +730,19 @@ def check_complete(self, cell: str):
return 'incomplete', find_last_indent(lines)

if (
tokens_by_line[-1][-1].type != tokenize.ENDMARKER
and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN
tokens_by_line[-1][-1].type != token.ENDMARKER
and tokens_by_line[-1][-1].type != token.ERRORTOKEN
):
# We're in a multiline string or expression
return 'incomplete', find_last_indent(lines)

newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
newline_types = {token.NEWLINE, token.COMMENT, token.ENDMARKER} # type: ignore

# Pop the last line which only contains DEDENTs and ENDMARKER
last_token_line = None
if {t.type for t in tokens_by_line[-1]} in [
{tokenize.DEDENT, tokenize.ENDMARKER},
{tokenize.ENDMARKER}
{token.DEDENT, token.ENDMARKER},
{token.ENDMARKER}
] and len(tokens_by_line) > 1:
last_token_line = tokens_by_line.pop()

Expand All @@ -757,7 +755,7 @@ def check_complete(self, cell: str):
if tokens_by_line[-1][-1].string == ':':
# The last line starts a block (e.g. 'if foo:')
ix = 0
while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
while tokens_by_line[-1][ix].type in {token.INDENT, token.DEDENT}:
ix += 1

indent = tokens_by_line[-1][ix].start[1]
Expand All @@ -779,7 +777,7 @@ def check_complete(self, cell: str):
if res is None:
return 'incomplete', find_last_indent(lines)

if last_token_line and last_token_line[0].type == tokenize.DEDENT:
if last_token_line and last_token_line[0].type == token.DEDENT:
if ends_with_newline:
return 'complete', None
return 'incomplete', find_last_indent(lines)
Expand Down
Loading