Skip to content

gh-131878: Fix input of unicode characters with two or more code points in the REPL on Windows in vt mode #133030

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

gh-131878: Fix input of unicode characters with two or more code points in the REPL on Windows in vt mode #133030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a06c42a
Revert "gh-130804: Fix support of typing unicode chars in pyrepl (#13…
chris-eibl Apr 26, 2025
ab8efa7
fix vt on Windows
chris-eibl Apr 26, 2025
49078f3
only accept bytes of length=1 in BaseEventQueue.push()
chris-eibl Apr 26, 2025
f430f31
remove unused `push_char` occurrences
chris-eibl Apr 26, 2025
6fe04a5
self.push(_c.to_bytes())
chris-eibl Apr 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions Lib/_pyrepl/base_eventqueue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,19 +69,13 @@ def insert(self, event: Event) -> None:
trace('added event {event}', event=event)
self.events.append(event)

def push(self, char: int | bytes | str) -> None:
def push(self, char: bytes) -> None:
"""
Processes a character by updating the buffer and handling special key mappings.
"""
ord_char = char if isinstance(char, int) else ord(char)
if ord_char > 255:
assert isinstance(char, str)
char = bytes(char.encode(self.encoding, "replace"))
self.buf.extend(char)
else:
char = bytes(bytearray((ord_char,)))
self.buf.append(ord_char)

assert isinstance(char, bytes)
assert len(char) == 1
self.buf.extend(char)
if char in self.keymap:
if self.keymap is self.compiled_keymap:
# sanity check, buffer is empty when a special key comes
Expand All @@ -102,7 +96,7 @@ def push(self, char: int | bytes | str) -> None:
self.keymap = self.compiled_keymap
self.insert(Event('key', '\033', bytearray(b'\033')))
for _c in self.flush_buf()[1:]:
self.push(_c)
self.push(_c.to_bytes())

else:
try:
Expand Down
7 changes: 0 additions & 7 deletions Lib/_pyrepl/console.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,6 @@ def get_event(self, block: bool = True) -> Event | None:
completion of an event."""
...

@abstractmethod
def push_char(self, char: int | bytes) -> None:
"""
Push a character to the console event queue.
"""
...

@abstractmethod
def beep(self) -> None: ...

Expand Down
4 changes: 0 additions & 4 deletions Lib/_pyrepl/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -733,10 +733,6 @@ def handle1(self, block: bool = True) -> bool:
self.do_cmd(cmd)
return True

def push_char(self, char: int | bytes) -> None:
self.console.push_char(char)
self.handle1(block=False)

def readline(self, startup_hook: Callback | None = None) -> str:
"""Read a line. The implementation of this method also shows
how to drive Reader if you want more control over the event
Expand Down
2 changes: 1 addition & 1 deletion Lib/_pyrepl/unix_console.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ def restore(self):
signal.signal(signal.SIGWINCH, self.old_sigwinch)
del self.old_sigwinch

def push_char(self, char: int | bytes) -> None:
def push_char(self, char: bytes) -> None:
"""
Push a character to the console event queue.
"""
Expand Down
10 changes: 3 additions & 7 deletions Lib/_pyrepl/windows_console.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,9 @@ def get_event(self, block: bool = True) -> Event | None:
return None
elif self.__vt_support:
# If virtual terminal is enabled, scanning VT sequences
self.event_queue.push(rec.Event.KeyEvent.uChar.UnicodeChar)
for char in raw_key.encode(self.event_queue.encoding,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd love to use PEP-467 iterbytes() here :)

"replace"):
self.event_queue.push(char.to_bytes())
continue

if key_event.dwControlKeyState & ALT_ACTIVE:
Expand All @@ -479,12 +481,6 @@ def get_event(self, block: bool = True) -> Event | None:
return Event(evt="key", data=key, raw=raw_key)
return self.event_queue.get()

def push_char(self, char: int | bytes) -> None:
"""
Push a character to the console event queue.
"""
raise NotImplementedError("push_char not supported on Windows")

def beep(self) -> None:
self.__write("\x07")

Expand Down
3 changes: 0 additions & 3 deletions Lib/test/test_pyrepl/support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,6 @@ def move_cursor(self, x: int, y: int) -> None:
def set_cursor_vis(self, visible: bool) -> None:
pass

def push_char(self, char: int | bytes) -> None:
pass

def beep(self) -> None:
pass

Expand Down
43 changes: 25 additions & 18 deletions Lib/test/test_pyrepl/test_eventqueue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def test_push_with_key_in_keymap(self, mock_keymap):
mock_keymap.compile_keymap.return_value = {"a": "b"}
eq = self.make_eventqueue()
eq.keymap = {b"a": "b"}
eq.push("a")
eq.push(b"a")
mock_keymap.compile_keymap.assert_called()
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "b")
Expand All @@ -63,7 +63,7 @@ def test_push_without_key_in_keymap(self, mock_keymap):
mock_keymap.compile_keymap.return_value = {"a": "b"}
eq = self.make_eventqueue()
eq.keymap = {b"c": "d"}
eq.push("a")
eq.push(b"a")
mock_keymap.compile_keymap.assert_called()
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "a")
Expand All @@ -73,13 +73,13 @@ def test_push_with_keymap_in_keymap(self, mock_keymap):
mock_keymap.compile_keymap.return_value = {"a": "b"}
eq = self.make_eventqueue()
eq.keymap = {b"a": {b"b": "c"}}
eq.push("a")
eq.push(b"a")
mock_keymap.compile_keymap.assert_called()
self.assertTrue(eq.empty())
eq.push("b")
eq.push(b"b")
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "c")
eq.push("d")
eq.push(b"d")
self.assertEqual(eq.events[1].evt, "key")
self.assertEqual(eq.events[1].data, "d")

Expand All @@ -88,32 +88,32 @@ def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap):
mock_keymap.compile_keymap.return_value = {"a": "b"}
eq = self.make_eventqueue()
eq.keymap = {b"a": {b"b": "c"}}
eq.push("a")
eq.push(b"a")
mock_keymap.compile_keymap.assert_called()
self.assertTrue(eq.empty())
eq.flush_buf()
eq.push("\033")
eq.push(b"\033")
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "\033")
eq.push("b")
eq.push(b"b")
self.assertEqual(eq.events[1].evt, "key")
self.assertEqual(eq.events[1].data, "b")

def test_push_special_key(self):
eq = self.make_eventqueue()
eq.keymap = {}
eq.push("\x1b")
eq.push("[")
eq.push("A")
eq.push(b"\x1b")
eq.push(b"[")
eq.push(b"A")
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "\x1b")

def test_push_unrecognized_escape_sequence(self):
eq = self.make_eventqueue()
eq.keymap = {}
eq.push("\x1b")
eq.push("[")
eq.push("Z")
eq.push(b"\x1b")
eq.push(b"[")
eq.push(b"Z")
self.assertEqual(len(eq.events), 3)
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "\x1b")
Expand All @@ -122,12 +122,19 @@ def test_push_unrecognized_escape_sequence(self):
self.assertEqual(eq.events[2].evt, "key")
self.assertEqual(eq.events[2].data, "Z")

def test_push_unicode_character(self):
def test_push_unicode_character_two_bytes(self):
eq = self.make_eventqueue()
eq.keymap = {}
eq.push("ч")
self.assertEqual(eq.events[0].evt, "key")
self.assertEqual(eq.events[0].data, "ч")

encoded_bytes = "ч".encode(eq.encoding, "replace")
self.assertEqual(len(encoded_bytes), 2)
eq.push(encoded_bytes[0].to_bytes())
self.assertEqual(eq.get(), None)

eq.push(encoded_bytes[1].to_bytes())
e = eq.get()
self.assertEqual(e.evt, "key")
self.assertEqual(e.data, "ч")


@unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows")
Expand Down
Loading