From a06c42afada3c972f6ffe48a8c649ee0bbc51646 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sat, 26 Apr 2025 23:08:28 +0200 Subject: [PATCH 1/5] Revert "gh-130804: Fix support of typing unicode chars in pyrepl (#130805)" This reverts commit 7c98b0674daa3e4eb3e8f35afb61a0dba61d1780. --- Lib/_pyrepl/base_eventqueue.py | 12 +++--------- Lib/test/test_pyrepl/test_eventqueue.py | 7 ------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index e018c4fc18308e..9cae1db112a838 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -69,19 +69,13 @@ def insert(self, event: Event) -> None: trace('added event {event}', event=event) self.events.append(event) - def push(self, char: int | bytes | str) -> None: + def push(self, char: int | bytes) -> None: """ Processes a character by updating the buffer and handling special key mappings. """ ord_char = char if isinstance(char, int) else ord(char) - if ord_char > 255: - assert isinstance(char, str) - char = bytes(char.encode(self.encoding, "replace")) - self.buf.extend(char) - else: - char = bytes(bytearray((ord_char,))) - self.buf.append(ord_char) - + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) if char in self.keymap: if self.keymap is self.compiled_keymap: # sanity check, buffer is empty when a special key comes diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index afb557103424a6..fb5bb6f8af09e7 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -122,13 +122,6 @@ def test_push_unrecognized_escape_sequence(self): self.assertEqual(eq.events[2].evt, "key") self.assertEqual(eq.events[2].data, "Z") - def test_push_unicode_character(self): - eq = self.make_eventqueue() - eq.keymap = {} - eq.push("ч") - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "ч") - @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): From ab8efa72d5dc5fe5ecf587da0f1bec5b49e68eeb Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sat, 26 Apr 2025 23:30:03 +0200 Subject: [PATCH 2/5] fix vt on Windows --- Lib/_pyrepl/windows_console.py | 4 +++- Lib/test/test_pyrepl/test_eventqueue.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 47fd3fd8f8909b..3359ff54eed19e 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -468,7 +468,9 @@ def get_event(self, block: bool = True) -> Event | None: return None elif self.__vt_support: # If virtual terminal is enabled, scanning VT sequences - self.event_queue.push(rec.Event.KeyEvent.uChar.UnicodeChar) + for char in raw_key.encode(self.event_queue.encoding, + "replace"): + self.event_queue.push(char.to_bytes()) continue if key_event.dwControlKeyState & ALT_ACTIVE: diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index fb5bb6f8af09e7..9ed8623bd4b32f 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -122,6 +122,20 @@ def test_push_unrecognized_escape_sequence(self): self.assertEqual(eq.events[2].evt, "key") self.assertEqual(eq.events[2].data, "Z") + def test_push_unicode_character_two_bytes(self): + eq = self.make_eventqueue() + eq.keymap = {} + + encoded_bytes = "ч".encode(eq.encoding, "replace") + self.assertEqual(len(encoded_bytes), 2) + eq.push(encoded_bytes[0].to_bytes()) + self.assertEqual(eq.get(), None) + + eq.push(encoded_bytes[1].to_bytes()) + e = eq.get() + self.assertEqual(e.evt, "key") + self.assertEqual(e.data, "ч") + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): From 49078f3114308f3a8b0fd6ecf03c8616a71b1b06 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sat, 26 Apr 2025 23:54:15 +0200 Subject: [PATCH 3/5] only accept bytes of length=1 in BaseEventQueue.push() --- Lib/_pyrepl/base_eventqueue.py | 8 +++---- Lib/test/test_pyrepl/test_eventqueue.py | 28 ++++++++++++------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 9cae1db112a838..cca95338490f28 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -69,13 +69,13 @@ def insert(self, event: Event) -> None: trace('added event {event}', event=event) self.events.append(event) - def push(self, char: int | bytes) -> None: + def push(self, char: bytes) -> None: """ Processes a character by updating the buffer and handling special key mappings. """ - ord_char = char if isinstance(char, int) else ord(char) - char = bytes(bytearray((ord_char,))) - self.buf.append(ord_char) + assert isinstance(char, bytes) + assert len(char) == 1 + self.buf.extend(char) if char in self.keymap: if self.keymap is self.compiled_keymap: # sanity check, buffer is empty when a special key comes diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index 9ed8623bd4b32f..cce0119eb6f076 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -53,7 +53,7 @@ def test_push_with_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": "b"} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "b") @@ -63,7 +63,7 @@ def test_push_without_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"c": "d"} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "a") @@ -73,13 +73,13 @@ def test_push_with_keymap_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) - eq.push("b") + eq.push(b"b") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "c") - eq.push("d") + eq.push(b"d") self.assertEqual(eq.events[1].evt, "key") self.assertEqual(eq.events[1].data, "d") @@ -88,32 +88,32 @@ def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} eq = self.make_eventqueue() eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") + eq.push(b"a") mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) eq.flush_buf() - eq.push("\033") + eq.push(b"\033") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\033") - eq.push("b") + eq.push(b"b") self.assertEqual(eq.events[1].evt, "key") self.assertEqual(eq.events[1].data, "b") def test_push_special_key(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("A") + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"A") self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\x1b") def test_push_unrecognized_escape_sequence(self): eq = self.make_eventqueue() eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("Z") + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"Z") self.assertEqual(len(eq.events), 3) self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "\x1b") From f430f31b2e1d15333371d79dee633540502dfb1e Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 27 Apr 2025 00:38:17 +0200 Subject: [PATCH 4/5] remove unused `push_char` occurrences --- Lib/_pyrepl/console.py | 7 ------- Lib/_pyrepl/reader.py | 4 ---- Lib/_pyrepl/unix_console.py | 2 +- Lib/_pyrepl/windows_console.py | 6 ------ Lib/test/test_pyrepl/support.py | 3 --- 5 files changed, 1 insertion(+), 21 deletions(-) diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 8956fb1242e52a..975f03b0c9245b 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -98,13 +98,6 @@ def get_event(self, block: bool = True) -> Event | None: completion of an event.""" ... - @abstractmethod - def push_char(self, char: int | bytes) -> None: - """ - Push a character to the console event queue. - """ - ... - @abstractmethod def beep(self) -> None: ... diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 7fc2422dac9c3f..21169ce421f0a3 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -733,10 +733,6 @@ def handle1(self, block: bool = True) -> bool: self.do_cmd(cmd) return True - def push_char(self, char: int | bytes) -> None: - self.console.push_char(char) - self.handle1(block=False) - def readline(self, startup_hook: Callback | None = None) -> str: """Read a line. The implementation of this method also shows how to drive Reader if you want more control over the event diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 96379bc20f3357..6d5291e6b373e1 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -380,7 +380,7 @@ def restore(self): signal.signal(signal.SIGWINCH, self.old_sigwinch) del self.old_sigwinch - def push_char(self, char: int | bytes) -> None: + def push_char(self, char: bytes) -> None: """ Push a character to the console event queue. """ diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 3359ff54eed19e..4e94b45de2ee52 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -481,12 +481,6 @@ def get_event(self, block: bool = True) -> Event | None: return Event(evt="key", data=key, raw=raw_key) return self.event_queue.get() - def push_char(self, char: int | bytes) -> None: - """ - Push a character to the console event queue. - """ - raise NotImplementedError("push_char not supported on Windows") - def beep(self) -> None: self.__write("\x07") diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index 3692e164cb9254..81def6f1b692ba 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -149,9 +149,6 @@ def move_cursor(self, x: int, y: int) -> None: def set_cursor_vis(self, visible: bool) -> None: pass - def push_char(self, char: int | bytes) -> None: - pass - def beep(self) -> None: pass From 6fe04a5bccb9282c4022c47199d25712f14696ff Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Sun, 27 Apr 2025 00:49:09 +0200 Subject: [PATCH 5/5] self.push(_c.to_bytes()) --- Lib/_pyrepl/base_eventqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index cca95338490f28..ba90002227785e 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -96,7 +96,7 @@ def push(self, char: bytes) -> None: self.keymap = self.compiled_keymap self.insert(Event('key', '\033', bytearray(b'\033'))) for _c in self.flush_buf()[1:]: - self.push(_c) + self.push(_c.to_bytes()) else: try: