Skip to content

Commit fac41f5

Browse files
ambvViicoshugovk
authored
gh-131507: Add support for syntax highlighting in PyREPL (GH-133247)
Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
1 parent bfcbb28 commit fac41f5

21 files changed

+654
-99
lines changed

‎Doc/whatsnew/3.14.rst

+17
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,23 @@ For further information on how to build Python, see
560560
(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this
561561
in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.)
562562

563+
Syntax highlighting in PyREPL
564+
-----------------------------
565+
566+
The default :term:`interactive` shell now highlights Python syntax as you
567+
type. The feature is enabled by default unless the
568+
:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling
569+
environment variables are used. See :ref:`using-on-controlling-color` for
570+
details.
571+
572+
The default color theme for syntax highlighting strives for good contrast
573+
and uses exclusively the 4-bit VGA standard ANSI color codes for maximum
574+
compatibility. The theme can be customized using an experimental API
575+
``_colorize.set_theme()``. This can be called interactively, as well as
576+
in the :envvar:`PYTHONSTARTUP` script.
577+
578+
(Contributed by Łukasz Langa in :gh:`131507`.)
579+
563580

564581
Other language changes
565582
======================

‎Lib/_colorize.py

+42-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,22 @@
77

88
# types
99
if False:
10-
from typing import IO
10+
from typing import IO, Literal
11+
12+
type ColorTag = Literal[
13+
"PROMPT",
14+
"KEYWORD",
15+
"BUILTIN",
16+
"COMMENT",
17+
"STRING",
18+
"NUMBER",
19+
"OP",
20+
"DEFINITION",
21+
"SOFT_KEYWORD",
22+
"RESET",
23+
]
24+
25+
theme: dict[ColorTag, str]
1126

1227

1328
class ANSIColors:
@@ -23,6 +38,7 @@ class ANSIColors:
2338
WHITE = "\x1b[37m" # more like LIGHT GRAY
2439
YELLOW = "\x1b[33m"
2540

41+
BOLD = "\x1b[1m"
2642
BOLD_BLACK = "\x1b[1;30m" # DARK GRAY
2743
BOLD_BLUE = "\x1b[1;34m"
2844
BOLD_CYAN = "\x1b[1;36m"
@@ -120,3 +136,28 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
120136
return os.isatty(file.fileno())
121137
except io.UnsupportedOperation:
122138
return hasattr(file, "isatty") and file.isatty()
139+
140+
141+
def set_theme(t: dict[ColorTag, str] | None = None) -> None:
142+
global theme
143+
144+
if t:
145+
theme = t
146+
return
147+
148+
colors = get_colors()
149+
theme = {
150+
"PROMPT": colors.BOLD_MAGENTA,
151+
"KEYWORD": colors.BOLD_BLUE,
152+
"BUILTIN": colors.CYAN,
153+
"COMMENT": colors.RED,
154+
"STRING": colors.GREEN,
155+
"NUMBER": colors.YELLOW,
156+
"OP": colors.RESET,
157+
"DEFINITION": colors.BOLD,
158+
"SOFT_KEYWORD": colors.BOLD_BLUE,
159+
"RESET": colors.RESET,
160+
}
161+
162+
163+
set_theme()

‎Lib/_pyrepl/_module_completer.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import pkgutil
44
import sys
5+
import token
56
import tokenize
67
from io import StringIO
78
from contextlib import contextmanager
@@ -180,8 +181,8 @@ class ImportParser:
180181
when parsing multiple statements.
181182
"""
182183
_ignored_tokens = {
183-
tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT,
184-
tokenize.NL, tokenize.NEWLINE, tokenize.ENDMARKER
184+
token.INDENT, token.DEDENT, token.COMMENT,
185+
token.NL, token.NEWLINE, token.ENDMARKER
185186
}
186187
_keywords = {'import', 'from', 'as'}
187188

@@ -350,11 +351,11 @@ def peek(self) -> TokenInfo | None:
350351
def peek_name(self) -> bool:
351352
if not (tok := self.peek()):
352353
return False
353-
return tok.type == tokenize.NAME
354+
return tok.type == token.NAME
354355

355356
def pop_name(self) -> str:
356357
tok = self.pop()
357-
if tok.type != tokenize.NAME:
358+
if tok.type != token.NAME:
358359
raise ParseError('pop_name')
359360
return tok.string
360361

‎Lib/_pyrepl/commands.py

+18-11
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
from __future__ import annotations
2323
import os
24+
import time
2425

2526
# Categories of actions:
2627
# killing
@@ -31,6 +32,7 @@
3132
# finishing
3233
# [completion]
3334

35+
from .trace import trace
3436

3537
# types
3638
if False:
@@ -471,19 +473,24 @@ def do(self) -> None:
471473

472474

473475
class paste_mode(Command):
474-
475476
def do(self) -> None:
476477
self.reader.paste_mode = not self.reader.paste_mode
477478
self.reader.dirty = True
478479

479480

480-
class enable_bracketed_paste(Command):
481-
def do(self) -> None:
482-
self.reader.paste_mode = True
483-
self.reader.in_bracketed_paste = True
484-
485-
class disable_bracketed_paste(Command):
486-
def do(self) -> None:
487-
self.reader.paste_mode = False
488-
self.reader.in_bracketed_paste = False
489-
self.reader.dirty = True
481+
class perform_bracketed_paste(Command):
482+
def do(self) -> None:
483+
done = "\x1b[201~"
484+
data = ""
485+
start = time.time()
486+
while done not in data:
487+
self.reader.console.wait(100)
488+
ev = self.reader.console.getpending()
489+
data += ev.data
490+
trace(
491+
"bracketed pasting of {l} chars done in {s:.2f}s",
492+
l=len(data),
493+
s=time.time() - start,
494+
)
495+
self.reader.insert(data.replace(done, ""))
496+
self.reader.last_refresh_cache.invalidated = True

‎Lib/_pyrepl/mypy.ini

-4
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,3 @@ check_untyped_defs = False
2323
# Various internal modules that typeshed deliberately doesn't have stubs for:
2424
[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
2525
ignore_missing_imports = True
26-
27-
# Other untyped parts of the stdlib
28-
[mypy-idlelib.*]
29-
ignore_missing_imports = True

‎Lib/_pyrepl/reader.py

+25-24
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,13 @@
2222
from __future__ import annotations
2323

2424
import sys
25+
import _colorize
2526

2627
from contextlib import contextmanager
2728
from dataclasses import dataclass, field, fields
28-
from _colorize import can_colorize, ANSIColors
29-
3029

3130
from . import commands, console, input
32-
from .utils import wlen, unbracket, disp_str
31+
from .utils import wlen, unbracket, disp_str, gen_colors
3332
from .trace import trace
3433

3534

@@ -38,8 +37,7 @@
3837
from .types import Callback, SimpleContextManager, KeySpec, CommandName
3938

4039

41-
# syntax classes:
42-
40+
# syntax classes
4341
SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
4442

4543

@@ -105,8 +103,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]:
105103
(r"\M-9", "digit-arg"),
106104
(r"\M-\n", "accept"),
107105
("\\\\", "self-insert"),
108-
(r"\x1b[200~", "enable_bracketed_paste"),
109-
(r"\x1b[201~", "disable_bracketed_paste"),
106+
(r"\x1b[200~", "perform-bracketed-paste"),
110107
(r"\x03", "ctrl-c"),
111108
]
112109
+ [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
@@ -144,16 +141,17 @@ class Reader:
144141
Instance variables of note include:
145142
146143
* buffer:
147-
A *list* (*not* a string at the moment :-) containing all the
148-
characters that have been entered.
144+
A per-character list containing all the characters that have been
145+
entered. Does not include color information.
149146
* console:
150147
Hopefully encapsulates the OS dependent stuff.
151148
* pos:
152149
A 0-based index into 'buffer' for where the insertion point
153150
is.
154151
* screeninfo:
155-
Ahem. This list contains some info needed to move the
156-
insertion point around reasonably efficiently.
152+
A list of screen position tuples. Each list element is a tuple
153+
representing information on visible line length for a given line.
154+
Allows for efficient skipping of color escape sequences.
157155
* cxy, lxy:
158156
the position of the insertion point in screen ...
159157
* syntax_table:
@@ -203,7 +201,6 @@ class Reader:
203201
dirty: bool = False
204202
finished: bool = False
205203
paste_mode: bool = False
206-
in_bracketed_paste: bool = False
207204
commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
208205
last_command: type[Command] | None = None
209206
syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
@@ -221,7 +218,6 @@ class Reader:
221218
## cached metadata to speed up screen refreshes
222219
@dataclass
223220
class RefreshCache:
224-
in_bracketed_paste: bool = False
225221
screen: list[str] = field(default_factory=list)
226222
screeninfo: list[tuple[int, list[int]]] = field(init=False)
227223
line_end_offsets: list[int] = field(default_factory=list)
@@ -235,7 +231,6 @@ def update_cache(self,
235231
screen: list[str],
236232
screeninfo: list[tuple[int, list[int]]],
237233
) -> None:
238-
self.in_bracketed_paste = reader.in_bracketed_paste
239234
self.screen = screen.copy()
240235
self.screeninfo = screeninfo.copy()
241236
self.pos = reader.pos
@@ -248,8 +243,7 @@ def valid(self, reader: Reader) -> bool:
248243
return False
249244
dimensions = reader.console.width, reader.console.height
250245
dimensions_changed = dimensions != self.dimensions
251-
paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
252-
return not (dimensions_changed or paste_changed)
246+
return not dimensions_changed
253247

254248
def get_cached_location(self, reader: Reader) -> tuple[int, int]:
255249
if self.invalidated:
@@ -279,7 +273,7 @@ def __post_init__(self) -> None:
279273
self.screeninfo = [(0, [])]
280274
self.cxy = self.pos2xy()
281275
self.lxy = (self.pos, 0)
282-
self.can_colorize = can_colorize()
276+
self.can_colorize = _colorize.can_colorize()
283277

284278
self.last_refresh_cache.screeninfo = self.screeninfo
285279
self.last_refresh_cache.pos = self.pos
@@ -316,6 +310,12 @@ def calc_screen(self) -> list[str]:
316310
pos -= offset
317311

318312
prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
313+
314+
if self.can_colorize:
315+
colors = list(gen_colors(self.get_unicode()))
316+
else:
317+
colors = None
318+
trace("colors = {colors}", colors=colors)
319319
lines = "".join(self.buffer[offset:]).split("\n")
320320
cursor_found = False
321321
lines_beyond_cursor = 0
@@ -343,9 +343,8 @@ def calc_screen(self) -> list[str]:
343343
screeninfo.append((0, []))
344344
pos -= line_len + 1
345345
prompt, prompt_len = self.process_prompt(prompt)
346-
chars, char_widths = disp_str(line)
346+
chars, char_widths = disp_str(line, colors, offset)
347347
wrapcount = (sum(char_widths) + prompt_len) // self.console.width
348-
trace("wrapcount = {wrapcount}", wrapcount=wrapcount)
349348
if wrapcount == 0 or not char_widths:
350349
offset += line_len + 1 # Takes all of the line plus the newline
351350
last_refresh_line_end_offsets.append(offset)
@@ -479,7 +478,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
479478
'lineno'."""
480479
if self.arg is not None and cursor_on_line:
481480
prompt = f"(arg: {self.arg}) "
482-
elif self.paste_mode and not self.in_bracketed_paste:
481+
elif self.paste_mode:
483482
prompt = "(paste) "
484483
elif "\n" in self.buffer:
485484
if lineno == 0:
@@ -492,7 +491,11 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
492491
prompt = self.ps1
493492

494493
if self.can_colorize:
495-
prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
494+
prompt = (
495+
f"{_colorize.theme["PROMPT"]}"
496+
f"{prompt}"
497+
f"{_colorize.theme["RESET"]}"
498+
)
496499
return prompt
497500

498501
def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
@@ -567,6 +570,7 @@ def insert(self, text: str | list[str]) -> None:
567570
def update_cursor(self) -> None:
568571
"""Move the cursor to reflect changes in self.pos"""
569572
self.cxy = self.pos2xy()
573+
trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy)
570574
self.console.move_cursor(*self.cxy)
571575

572576
def after_command(self, cmd: Command) -> None:
@@ -633,9 +637,6 @@ def update_screen(self) -> None:
633637

634638
def refresh(self) -> None:
635639
"""Recalculate and refresh the screen."""
636-
if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
637-
return
638-
639640
# this call sets up self.cxy, so call it first.
640641
self.screen = self.calc_screen()
641642
self.console.refresh(self.screen, self.cxy)

‎Lib/_pyrepl/readline.py

-4
Original file line numberDiff line numberDiff line change
@@ -276,10 +276,6 @@ def do(self) -> None:
276276
r = self.reader # type: ignore[assignment]
277277
r.dirty = True # this is needed to hide the completion menu, if visible
278278

279-
if self.reader.in_bracketed_paste:
280-
r.insert("\n")
281-
return
282-
283279
# if there are already several lines and the cursor
284280
# is not on the last one, always insert a new \n.
285281
text = r.get_unicode()

‎Lib/_pyrepl/simple_interact.py

-1
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,6 @@ def maybe_run_command(statement: str) -> bool:
157157
r.pos = len(r.get_unicode())
158158
r.dirty = True
159159
r.refresh()
160-
r.in_bracketed_paste = False
161160
console.write("\nKeyboardInterrupt\n")
162161
console.resetbuffer()
163162
except MemoryError:

0 commit comments

Comments
 (0)