From 62401b12f82d82b3e788d6ed24606f4cec820a6f Mon Sep 17 00:00:00 2001 From: harjoth Date: Wed, 8 Jul 2026 23:12:49 -0700 Subject: [PATCH 1/4] gh-152907: Restore cooked output flags around the input hook in the new REPL pyrepl clears OPOST for its own cursor rendering but calls PyOS_InputHook from inside the raw-mode read loop, so output written by an input hook (GUI toolkit event loops, and any warning/traceback/print they emit) is emitted with bare '\n' and no '\r'. Restore the terminal's saved output flags around the hook call and re-enter raw mode afterwards; only oflag is toggled so ECHO/ICANON stay off at the prompt. --- Lib/_pyrepl/unix_console.py | 19 +++- Lib/test/test_pyrepl/test_unix_console.py | 91 +++++++++++++++++++ ...-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst | 4 + 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 8edd1c36a7232d..0e7f8f6d47e6a0 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -491,6 +491,7 @@ def prepare(self) -> None: raw.cc[termios.VMIN] = b"\x01" raw.cc[termios.VTIME] = b"\x00" self.__input_fd_set(raw) + self.__rawtermstate = raw # Apple Terminal will re-wrap lines for us unless we preempt the # damage. @@ -731,7 +732,23 @@ def input_hook(self): # avoid inline imports here so the repl doesn't get flooded # with import logging from -X importtime=2 if posix is not None and posix._is_inputhook_installed(): - return posix._inputhook + return self.__run_input_hook + + def __run_input_hook(self): + # Input-hook callbacks (GUI toolkit event loops, and any warning, + # traceback or print they emit) expect normal cooked-mode output, but + # pyrepl runs with OPOST disabled so it can drive the cursor itself. + # Temporarily restore the terminal's saved output flags around the + # hook so that '\n' is translated to '\r\n' as it was under the + # readline REPL, then re-enter raw mode. Only oflag is toggled -- + # re-enabling ECHO/ICANON at the prompt would be wrong. See gh-152907. + cooked = self.__rawtermstate.copy() + cooked.oflag = self.__svtermstate.oflag + self.__input_fd_set(cooked) + try: + posix._inputhook() + finally: + self.__input_fd_set(self.__rawtermstate) def __enable_bracketed_paste(self) -> None: os.write(self.output_fd, b"\x1b[?2004h") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 2fc8398923cbf3..2d0b3c536d4f1a 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,9 +1,11 @@ import errno import itertools import os +import select import signal import sys import threading +import time import unittest from functools import partial from _colorize import ANSIColors @@ -424,3 +426,92 @@ def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): # EIO error should be handled gracefully in restore() console.restore() + + +try: + import pty + import termios as _termios +except ImportError: + pty = None + + +@unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") +@unittest.skipUnless(pty, "requires pty") +class TestUnixConsoleInputHook(TestCase): + # gh-152907: pyrepl runs with OPOST disabled so it can drive the cursor + # itself, but input-hook callbacks (GUI event loops, and any warning, + # traceback or print they emit) expect normal cooked-mode output. The + # console must restore cooked output around the hook call so that '\n' is + # translated back to '\r\n', then re-enter raw mode. + + def test_input_hook_output_is_cooked(self): + master_fd, slave_fd = pty.openpty() + + # Continuously drain the master side, like a real terminal. This is + # required because pyrepl switches the terminal mode with TCSADRAIN, + # which blocks until queued output has been consumed. + chunks = [] + reading = True + + def reader(): + while reading: + r, _, _ = select.select([master_fd], [], [], 0.1) + if master_fd in r: + try: + data = os.read(master_fd, 4096) + except OSError: + break + if not data: + break + chunks.append(data) + + reader_thread = threading.Thread(target=reader) + reader_thread.start() + + def cleanup(): + nonlocal reading + reading = False + reader_thread.join() + os.close(master_fd) + self.addCleanup(cleanup) + + # Establish a normal cooked terminal as the saved state so the fix has + # something to restore around the hook call. + attr = _termios.tcgetattr(slave_fd) + attr[1] |= _termios.OPOST | _termios.ONLCR + _termios.tcsetattr(slave_fd, _termios.TCSANOW, attr) + + console = UnixConsole(slave_fd, slave_fd, term="xterm") + console.prepare() + try: + # pyrepl's own rendering runs with OPOST cleared. + self.assertFalse(_termios.tcgetattr(slave_fd)[1] & _termios.OPOST) + + observed = {} + + def fake_hook(): + observed["oflag"] = _termios.tcgetattr(slave_fd)[1] + os.write(slave_fd, b"line1\nline2\n") + return 0 + + with patch("_pyrepl.unix_console.posix") as mock_posix: + mock_posix._is_inputhook_installed.return_value = True + mock_posix._inputhook.side_effect = fake_hook + hook = console.input_hook + self.assertIsNotNone(hook) + hook() + + # The hook ran with cooked output (OPOST on)... + self.assertTrue(observed["oflag"] & _termios.OPOST) + # ...and raw mode was restored afterwards. + self.assertFalse(_termios.tcgetattr(slave_fd)[1] & _termios.OPOST) + finally: + console.restore() + os.close(slave_fd) + + # Give the reader a moment to drain the hook's output. + time.sleep(0.2) + data = b"".join(chunks) + # The tty translated the hook's bare '\n' into '\r\n'. + self.assertIn(b"line1\r\nline2\r\n", data) + self.assertNotIn(b"line1\nline2\n", data) diff --git a/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst b/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst new file mode 100644 index 00000000000000..33247f0f3a3b96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-23-20-00.gh-issue-152907.oyPV9Y.rst @@ -0,0 +1,4 @@ +Restore cooked-mode terminal output flags around :c:data:`PyOS_InputHook` +callbacks in the new :term:`REPL` (:mod:`!_pyrepl`), so that output written +by an input hook (for example a GUI toolkit event loop) is no longer emitted +with ``OPOST`` disabled and keeps its carriage returns. From 1146b70fba6fb2516242bd9593b44a3d2b6a1c6c Mon Sep 17 00:00:00 2001 From: harjoth Date: Thu, 9 Jul 2026 17:51:17 -0700 Subject: [PATCH 2/4] Skip the input-hook test on platforms without pty devices The Emscripten buildbot has the pty module but no pty devices, so pty.openpty() raises OSError("out of pty devices"). Guard the test class the same way Lib/test/test_pty.py does. --- Lib/test/test_pyrepl/test_unix_console.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 2d0b3c536d4f1a..905ac8ac8e0ae4 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -10,6 +10,7 @@ from functools import partial from _colorize import ANSIColors from test.support import force_color, os_helper, force_not_colorized_test_class +from test.support import is_android, is_apple_mobile, is_wasm32 from test.support import threading_helper from unittest import TestCase @@ -437,6 +438,8 @@ def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): @unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") @unittest.skipUnless(pty, "requires pty") +@unittest.skipIf(is_android or is_apple_mobile or is_wasm32, + "pty is not available on this platform") class TestUnixConsoleInputHook(TestCase): # gh-152907: pyrepl runs with OPOST disabled so it can drive the cursor # itself, but input-hook callbacks (GUI event loops, and any warning, From a979156fdb14a1a9e7a524ad8a2eeaf77c0d8b21 Mon Sep 17 00:00:00 2001 From: harjoth Date: Sat, 11 Jul 2026 09:06:22 -0700 Subject: [PATCH 3/4] Propagate the input hook's return value and drop the sleep from the test Co-Authored-By: Claude Fable 5 --- Lib/_pyrepl/unix_console.py | 2 +- Lib/test/test_pyrepl/test_unix_console.py | 27 +++++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 0e7f8f6d47e6a0..1a61b4a425767c 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -746,7 +746,7 @@ def __run_input_hook(self): cooked.oflag = self.__svtermstate.oflag self.__input_fd_set(cooked) try: - posix._inputhook() + return posix._inputhook() finally: self.__input_fd_set(self.__rawtermstate) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index 905ac8ac8e0ae4..e4619062f915bf 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -5,7 +5,6 @@ import signal import sys import threading -import time import unittest from functools import partial from _colorize import ANSIColors @@ -450,9 +449,11 @@ class TestUnixConsoleInputHook(TestCase): def test_input_hook_output_is_cooked(self): master_fd, slave_fd = pty.openpty() - # Continuously drain the master side, like a real terminal. This is - # required because pyrepl switches the terminal mode with TCSADRAIN, - # which blocks until queued output has been consumed. + # Drain the master side continuously, like a real terminal would. + # This cannot be deferred until after the hook call: pyrepl switches + # the terminal mode with TCSADRAIN, which on some platforms (e.g. + # macOS) blocks until the queued output has actually been read from + # the master side, so an undrained pty deadlocks the mode switch. chunks = [] reading = True @@ -502,7 +503,7 @@ def fake_hook(): mock_posix._inputhook.side_effect = fake_hook hook = console.input_hook self.assertIsNotNone(hook) - hook() + self.assertEqual(hook(), 0) # The hook ran with cooked output (OPOST on)... self.assertTrue(observed["oflag"] & _termios.OPOST) @@ -512,8 +513,20 @@ def fake_hook(): console.restore() os.close(slave_fd) - # Give the reader a moment to drain the hook's output. - time.sleep(0.2) + # No sleep needed: the TCSADRAIN switch back to raw mode did not + # return until the hook's output was drained, so after joining the + # reader and emptying the master nothing is left in flight. + reading = False + reader_thread.join() + while select.select([master_fd], [], [], 0)[0]: + try: + extra = os.read(master_fd, 4096) + except OSError: + break + if not extra: + break + chunks.append(extra) + data = b"".join(chunks) # The tty translated the hook's bare '\n' into '\r\n'. self.assertIn(b"line1\r\nline2\r\n", data) From 3436671d8be8ae6d34d708470055b42b6f883836 Mon Sep 17 00:00:00 2001 From: harjoth Date: Sat, 11 Jul 2026 11:22:49 -0700 Subject: [PATCH 4/4] Trim comments Co-Authored-By: Claude Fable 5 --- Lib/_pyrepl/unix_console.py | 10 +++------- Lib/test/test_pyrepl/test_unix_console.py | 23 ++++++++--------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 1a61b4a425767c..8796cb19d05820 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -735,13 +735,9 @@ def input_hook(self): return self.__run_input_hook def __run_input_hook(self): - # Input-hook callbacks (GUI toolkit event loops, and any warning, - # traceback or print they emit) expect normal cooked-mode output, but - # pyrepl runs with OPOST disabled so it can drive the cursor itself. - # Temporarily restore the terminal's saved output flags around the - # hook so that '\n' is translated to '\r\n' as it was under the - # readline REPL, then re-enter raw mode. Only oflag is toggled -- - # re-enabling ECHO/ICANON at the prompt would be wrong. See gh-152907. + # gh-152907: input hooks expect cooked output, but pyrepl runs with + # OPOST disabled. Restore the saved output flags around the hook + # (only oflag; input must stay raw at the prompt). cooked = self.__rawtermstate.copy() cooked.oflag = self.__svtermstate.oflag self.__input_fd_set(cooked) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index e4619062f915bf..1897c735671785 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -440,20 +440,15 @@ def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): @unittest.skipIf(is_android or is_apple_mobile or is_wasm32, "pty is not available on this platform") class TestUnixConsoleInputHook(TestCase): - # gh-152907: pyrepl runs with OPOST disabled so it can drive the cursor - # itself, but input-hook callbacks (GUI event loops, and any warning, - # traceback or print they emit) expect normal cooked-mode output. The - # console must restore cooked output around the hook call so that '\n' is - # translated back to '\r\n', then re-enter raw mode. + # gh-152907: the console must restore cooked output (OPOST) around + # input-hook calls, then re-enter raw mode. def test_input_hook_output_is_cooked(self): master_fd, slave_fd = pty.openpty() - # Drain the master side continuously, like a real terminal would. - # This cannot be deferred until after the hook call: pyrepl switches - # the terminal mode with TCSADRAIN, which on some platforms (e.g. - # macOS) blocks until the queued output has actually been read from - # the master side, so an undrained pty deadlocks the mode switch. + # Drain the master continuously: on some platforms (e.g. macOS) + # tcsetattr(TCSADRAIN) blocks until the master side is read, so an + # undrained pty would deadlock the mode switch. chunks = [] reading = True @@ -479,8 +474,7 @@ def cleanup(): os.close(master_fd) self.addCleanup(cleanup) - # Establish a normal cooked terminal as the saved state so the fix has - # something to restore around the hook call. + # Start from a cooked terminal so there are saved flags to restore. attr = _termios.tcgetattr(slave_fd) attr[1] |= _termios.OPOST | _termios.ONLCR _termios.tcsetattr(slave_fd, _termios.TCSANOW, attr) @@ -513,9 +507,8 @@ def fake_hook(): console.restore() os.close(slave_fd) - # No sleep needed: the TCSADRAIN switch back to raw mode did not - # return until the hook's output was drained, so after joining the - # reader and emptying the master nothing is left in flight. + # The switch back to raw mode already drained the hook's output, so + # joining the reader is enough -- no sleep needed. reading = False reader_thread.join() while select.select([master_fd], [], [], 0)[0]: