From 998fc4a973b2b5ac925d13e01693360c24803330 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 24 Jul 2026 12:50:09 -0700 Subject: [PATCH 1/4] GH-142035: Fix wrapping of colorized argparse help text (#154634) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/argparse.py | 30 +++++++++- Lib/test/test_argparse.py | 56 +++++++++++++++++++ ...-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst | 1 + 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 29e6ebb9634261a..42891516b3bb2a0 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -776,7 +776,35 @@ def _split_lines(self, text, width): # The textwrap module is used only for formatting help. # Delay its import for speeding up the common usage of argparse. import textwrap - return textwrap.wrap(text, width) + decolored = self._decolor(text) + if decolored == text: + return textwrap.wrap(text, width) + + # gh-142035: colors inflate textwrap's length counts, so wrap + # the decolored text and re-apply colors per word; if textwrap + # split a word, keep the plain lines (colors can't be mapped). + plain = self._whitespace_matcher.sub(' ', decolored).strip() + if not plain: + # nothing visible to wrap (e.g. an empty interpolated value) + return [text] + plain_lines = textwrap.wrap(plain, width) + plain_words = plain.split() + colored_words = text.split() + # Drop escape-only tokens (e.g. an empty interpolated value). + if len(colored_words) != len(plain_words): + colored_words = [ + word for word in colored_words if self._decolor(word) + ] + colored_lines = [] + start = 0 + for plain_line in plain_lines: + plain_line_words = plain_line.split() + end = start + len(plain_line_words) + if plain_words[start:end] != plain_line_words: + return plain_lines + colored_lines.append(' '.join(colored_words[start:end])) + start = end + return colored_lines def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 1dc3f538f4ad8ba..fbeb933841129e3 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7580,6 +7580,62 @@ def test_argparse_color_custom_usage(self): ), ) + def test_argparse_color_wrapping_matches_uncolored(self): + # gh-142035: color codes must not affect where help text wraps. + # Stripping the escapes from colored help must yield exactly the + # same text as the uncolored help across representative widths. + def build(color, path="output.txt"): + parser = argparse.ArgumentParser(prog="PROG", color=color) + parser.add_argument( + "--mode", + default="auto", + choices=("auto", "fast", "slow"), + help="select the operating mode from the available choices " + "%(choices)s and note the default is %(default)s here", + ) + parser.add_argument( + "--path", + default=path, + help="write output to %(default)s and continue processing", + ) + return parser + + env = self.enterContext(os_helper.EnvironmentVarGuard()) + paths = ( + "output.txt", + "/var/lib/application/cache/unusually_long_generated_filename", + "production-read-only-replica", + ) + for path in paths: + for columns in ("80", "60", "45", "30", "20"): + with self.subTest(path=path, columns=columns): + env["COLUMNS"] = columns + colored = build(color=True, path=path).format_help() + plain = build(color=False, path=path).format_help() + self.assertIn( + f"{self.theme.interpolated_value}auto" + f"{self.theme.reset}", + colored, + ) + self.assertEqual(_colorize.decolor(colored), plain) + + def test_argparse_color_preserved_when_wrapping_between_words(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument( + "--mode", default="auto", + help="select the %(default)s operating mode from the available " + "options and continue with several more words", + ) + + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env["COLUMNS"] = "40" + help_text = parser.format_help() + + self.assertIn( + f"{self.theme.interpolated_value}auto{self.theme.reset}", + help_text, + ) + def test_custom_formatter_function(self): def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, indent_increment=5) diff --git a/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst b/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst new file mode 100644 index 000000000000000..f11f3a9e55200d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst @@ -0,0 +1 @@ +Fix incorrect wrapping of :mod:`argparse` help text when color is enabled. From f717b25712064b88e6dac5a919dbc717e69ebb9c Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 24 Jul 2026 13:33:26 -0700 Subject: [PATCH 2/4] GH-154638: Use lazy imports in argparse (#154639) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/argparse.py | 25 +++++++------------ Lib/test/test_argparse.py | 15 +++++++---- ...-07-24-17-02-46.gh-issue-154638.7YnFHH.rst | 1 + 3 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 42891516b3bb2a0..f8739d031e01c58 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -87,12 +87,17 @@ import os as _os -import re as _re import sys as _sys -from gettext import gettext as _ -from gettext import ngettext lazy import _colorize +lazy import copy +lazy import difflib +lazy import re as _re +lazy import shutil +lazy import textwrap +lazy import warnings +lazy from gettext import gettext as _ +lazy from gettext import ngettext SUPPRESS = '==SUPPRESS==' @@ -143,10 +148,8 @@ def _copy_items(items): return [] # The copy module is used only in the 'append' and 'append_const' # actions, and it is needed only when the default value isn't a list. - # Delay its import for speeding up the common case. if type(items) is list: return items[:] - import copy return copy.copy(items) @@ -186,7 +189,6 @@ def __init__( ): # default setting for width if width is None: - import shutil width = shutil.get_terminal_size().columns width -= 2 @@ -773,9 +775,6 @@ def _iter_indented_subactions(self, action): def _split_lines(self, text, width): text = self._whitespace_matcher.sub(' ', text).strip() - # The textwrap module is used only for formatting help. - # Delay its import for speeding up the common usage of argparse. - import textwrap decolored = self._decolor(text) if decolored == text: return textwrap.wrap(text, width) @@ -808,7 +807,6 @@ def _split_lines(self, text, width): def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() - import textwrap return textwrap.fill(text, width, initial_indent=indent, subsequent_indent=indent) @@ -1486,7 +1484,6 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): - import warnings warnings.warn( "FileType is deprecated. Simply open files after parsing arguments.", category=PendingDeprecationWarning, @@ -1893,7 +1890,6 @@ class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): if 'prefix_chars' in kwargs: - import warnings depr_msg = ( "The use of the undocumented 'prefix_chars' parameter in " "ArgumentParser.add_argument_group() is deprecated." @@ -2824,7 +2820,6 @@ def _check_value(self, action, value): if self.suggest_on_error and isinstance(value, str): if all(isinstance(choice, str) for choice in action.choices): - import difflib suggestions = difflib.get_close_matches(value, action.choices, 1) if suggestions: args['closest'] = suggestions[0] @@ -2964,8 +2959,6 @@ def _warning(self, message): def __getattr__(name): if name == "__version__": - from warnings import _deprecated - - _deprecated("__version__", remove=(3, 20)) + warnings._deprecated("__version__", remove=(3, 20)) return "1.1" # Do not change raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index fbeb933841129e3..442cbb1aaadfe3f 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3,7 +3,6 @@ import _colorize import contextlib import functools -import inspect import io import operator import os @@ -12,6 +11,7 @@ import sys import textwrap import tempfile +import types import unittest import argparse import warnings @@ -85,6 +85,8 @@ class TestLazyImports(unittest.TestCase): "_colorize", "copy", "difflib", + "gettext", + "re", "shutil", "textwrap", "warnings", @@ -99,7 +101,7 @@ def test_create_parser(self): # Test imports are still unused after # creating a parser create_parser = "argparse.ArgumentParser()" - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", @@ -114,7 +116,7 @@ def test_add_subparser(self): parser.add_subparsers(dest='command', required=False) """ ) - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", @@ -132,7 +134,7 @@ def test_parse_args(self): parser.parse_args(['BAR', '--foo', 'FOO']) """ ) - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", self.LAZY_IMPORTS - imported_modules, @@ -7098,7 +7100,10 @@ def test_all_exports_everything_but_modules(self): name for name, value in vars(argparse).items() if not (name.startswith("_") or name == 'ngettext') - if not inspect.ismodule(value) + if not isinstance( + value, + (types.ModuleType, types.LazyImportType), + ) ] self.assertEqual(sorted(items), sorted(argparse.__all__)) diff --git a/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst b/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst new file mode 100644 index 000000000000000..91c22640deba6b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst @@ -0,0 +1 @@ +Improve import time of :mod:`argparse` by lazily importing several dependencies. From 1a742d403243ac9915ffcc54e29516e58c44dc42 Mon Sep 17 00:00:00 2001 From: Brij Kapadia <97006829+brijkapadia@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:32:07 -0400 Subject: [PATCH 3/4] gh-154189: Fix use-after-free in `functools.partial_vectorcall` (GH-154508) --- Lib/test/test_functools.py | 34 ++++++++++ ...-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst | 4 ++ Modules/_functoolsmodule.c | 62 ++++++++++--------- 3 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 941dd7249a48d91..b46a1ce6d634a5f 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -579,6 +579,40 @@ def f(**kwargs): with self.assertRaises(RuntimeError): result = p(**{BadStr("poison"): "new_value"}) + def test_call_safety_against_reentrant_mutation(self): + def old_function(*args, **kwargs): + return "old_function", args, kwargs + + def new_function(*args, **kwargs): + return "new_function", args, kwargs + + g_partial = None + + class EvilKey(str): + armed = False + def __hash__(self): + if EvilKey.armed and g_partial is not None: + EvilKey.armed = False + new_args_tuple = ("new_arg",) + new_keywords_dict = {"new_keyword": None} + new_tuple_state = (new_function, new_args_tuple, new_keywords_dict, None) + g_partial.__setstate__(new_tuple_state) + gc.collect() + return str.__hash__(self) + + g_partial = functools.partial(old_function, "old_arg", old_keyword=None) + + kwargs = {EvilKey("evil_key"): None} + EvilKey.armed = True + + result = g_partial(**kwargs) + expected = ("old_function", ("old_arg",), {"old_keyword": None, "evil_key": None}) + self.assertEqual(result, expected) + + result = g_partial() + expected = ("new_function", ("new_arg",), {"new_keyword": None}) + self.assertEqual(result, expected) + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: diff --git a/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst b/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst new file mode 100644 index 000000000000000..3a745a7c5ce67d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst @@ -0,0 +1,4 @@ +Fixed a potential use-after-free when calling :func:`functools.partial`. +Now, when invoking a :func:`~functools.partial` object, the stored function, +positional arguments, and keyword arguments are preserved for the duration +of the call in case of reentrancy. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index b4595c55d519b93..1ab230218124a46 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -382,9 +382,14 @@ partial_vectorcall(PyObject *self, PyObject *const *args, return NULL; } - PyObject **pto_args = _PyTuple_ITEMS(pto->args); - Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); - Py_ssize_t pto_nkwds = PyDict_GET_SIZE(pto->kw); + PyObject *result = NULL; + PyObject *partial_function = Py_NewRef(pto->fn); + PyObject *partial_args = Py_NewRef(pto->args); + PyObject *partial_keywords = Py_NewRef(pto->kw); + + PyObject **pto_args = _PyTuple_ITEMS(partial_args); + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(partial_args); + Py_ssize_t pto_nkwds = PyDict_GET_SIZE(partial_keywords); Py_ssize_t nkwds = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); Py_ssize_t nargskw = nargs + nkwds; @@ -392,8 +397,9 @@ partial_vectorcall(PyObject *self, PyObject *const *args, if (!pto_nkwds) { /* Fast path if we're called without arguments */ if (nargskw == 0) { - return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, - pto_nargs, NULL); + result = _PyObject_VectorcallTstate(tstate, partial_function, pto_args, + pto_nargs, NULL); + goto done; } /* Use PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single @@ -402,10 +408,10 @@ partial_vectorcall(PyObject *self, PyObject *const *args, PyObject **newargs = (PyObject **)args - 1; PyObject *tmp = newargs[0]; newargs[0] = pto_args[0]; - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, newargs, - nargs + 1, kwnames); + result = _PyObject_VectorcallTstate(tstate, partial_function, newargs, + nargs + 1, kwnames); newargs[0] = tmp; - return ret; + goto done; } } @@ -435,7 +441,8 @@ partial_vectorcall(PyObject *self, PyObject *const *args, else { stack = PyMem_Malloc(init_stack_size * sizeof(PyObject *)); if (stack == NULL) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto done; } } @@ -457,20 +464,20 @@ partial_vectorcall(PyObject *self, PyObject *const *args, for (Py_ssize_t i = 0; i < nkwds; ++i) { key = PyTuple_GET_ITEM(kwnames, i); val = args[nargs + i]; - int contains = PyDict_Contains(pto->kw, key); + int contains = PyDict_Contains(partial_keywords, key); if (contains < 0) { - goto error; + goto clean_stack; } else if (contains == 1) { if (pto_kw_merged == NULL) { - pto_kw_merged = PyDict_Copy(pto->kw); + pto_kw_merged = PyDict_Copy(partial_keywords); if (pto_kw_merged == NULL) { - goto error; + goto clean_stack; } } if (PyDict_SetItem(pto_kw_merged, key, val) < 0) { Py_DECREF(pto_kw_merged); - goto error; + goto clean_stack; } } else { @@ -486,7 +493,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, tot_kwnames = PyTuple_New(tot_nkwds - n_merges); if (tot_kwnames == NULL) { Py_XDECREF(pto_kw_merged); - goto error; + goto clean_stack; } for (Py_ssize_t i = 0; i < n_tail; ++i) { key = Py_NewRef(stack[tot_nargskw + i]); @@ -496,7 +503,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, /* Copy pto_keywords with overlapping call keywords merged * Note, tail is already coppied. */ Py_ssize_t pos = 0, i = 0; - PyObject *keyword_dict = n_merges ? pto_kw_merged : pto->kw; + PyObject *keyword_dict = n_merges ? pto_kw_merged : partial_keywords; Py_BEGIN_CRITICAL_SECTION(keyword_dict); while (PyDict_Next(keyword_dict, &pos, &key, &val)) { assert(i < pto_nkwds); @@ -515,10 +522,8 @@ partial_vectorcall(PyObject *self, PyObject *const *args, tmp_stack = PyMem_Realloc(stack, (tot_nargskw - n_merges) * sizeof(PyObject *)); if (tmp_stack == NULL) { Py_DECREF(tot_kwnames); - if (stack != small_stack) { - PyMem_Free(stack); - } - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto clean_stack; } stack = tmp_stack; } @@ -547,21 +552,22 @@ partial_vectorcall(PyObject *self, PyObject *const *args, memcpy(stack + pto_nargs, args, nargs * sizeof(PyObject*)); } - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, - tot_nargs, tot_kwnames); - if (stack != small_stack) { - PyMem_Free(stack); - } + result = _PyObject_VectorcallTstate(tstate, partial_function, stack, + tot_nargs, tot_kwnames); if (pto_nkwds) { Py_DECREF(tot_kwnames); } - return ret; - error: + clean_stack: if (stack != small_stack) { PyMem_Free(stack); } - return NULL; + + done: + Py_DECREF(partial_function); + Py_DECREF(partial_args); + Py_DECREF(partial_keywords); + return result; } /* Set pto->vectorcall depending on the parameters of the partial object */ From 9560bd8f3533ac16684dfe11283bb834af52f10f Mon Sep 17 00:00:00 2001 From: Ivy Xu Date: Sat, 25 Jul 2026 06:33:29 +0800 Subject: [PATCH 4/4] gh-134745: Remove dead code in `thread_nt.h` left by gh-134747 (#154338) --- Python/thread_nt.h | 136 --------------------------------------------- 1 file changed, 136 deletions(-) diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9a29d14ef67678d..086dab912ecfd71 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -11,142 +11,6 @@ #include #endif -/* options */ -#ifndef _PY_USE_CV_LOCKS -#define _PY_USE_CV_LOCKS 1 /* use locks based on cond vars */ -#endif - -/* Now, define a non-recursive mutex using either condition variables - * and critical sections (fast) or using operating system mutexes - * (slow) - */ - -#if _PY_USE_CV_LOCKS - -#include "condvar.h" - -typedef struct _NRMUTEX -{ - PyMUTEX_T cs; - PyCOND_T cv; - int locked; -} NRMUTEX; -typedef NRMUTEX *PNRMUTEX; - -static PNRMUTEX -AllocNonRecursiveMutex(void) -{ - PNRMUTEX m = (PNRMUTEX)PyMem_RawMalloc(sizeof(NRMUTEX)); - if (!m) - return NULL; - if (PyCOND_INIT(&m->cv)) - goto fail; - if (PyMUTEX_INIT(&m->cs)) { - PyCOND_FINI(&m->cv); - goto fail; - } - m->locked = 0; - return m; -fail: - PyMem_RawFree(m); - return NULL; -} - -static VOID -FreeNonRecursiveMutex(PNRMUTEX mutex) -{ - if (mutex) { - PyCOND_FINI(&mutex->cv); - PyMUTEX_FINI(&mutex->cs); - PyMem_RawFree(mutex); - } -} - -static DWORD -EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) -{ - DWORD result = WAIT_OBJECT_0; - if (PyMUTEX_LOCK(&mutex->cs)) - return WAIT_FAILED; - if (milliseconds == INFINITE) { - while (mutex->locked) { - if (PyCOND_WAIT(&mutex->cv, &mutex->cs)) { - result = WAIT_FAILED; - break; - } - } - } else if (milliseconds != 0) { - /* wait at least until the deadline */ - PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); - PyTime_t deadline = _PyDeadline_Init(timeout); - while (mutex->locked) { - PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_TIMEOUT); - if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { - result = WAIT_FAILED; - break; - } - - timeout = _PyDeadline_Get(deadline); - if (timeout <= 0) { - break; - } - } - } - if (!mutex->locked) { - mutex->locked = 1; - result = WAIT_OBJECT_0; - } else if (result == WAIT_OBJECT_0) - result = WAIT_TIMEOUT; - /* else, it is WAIT_FAILED */ - PyMUTEX_UNLOCK(&mutex->cs); /* must ignore result here */ - return result; -} - -static BOOL -LeaveNonRecursiveMutex(PNRMUTEX mutex) -{ - BOOL result; - if (PyMUTEX_LOCK(&mutex->cs)) - return FALSE; - mutex->locked = 0; - /* condvar APIs return 0 on success. We need to return TRUE on success. */ - result = !PyCOND_SIGNAL(&mutex->cv); - PyMUTEX_UNLOCK(&mutex->cs); - return result; -} - -#else /* if ! _PY_USE_CV_LOCKS */ - -/* NR-locks based on a kernel mutex */ -#define PNRMUTEX HANDLE - -static PNRMUTEX -AllocNonRecursiveMutex(void) -{ - return CreateSemaphore(NULL, 1, 1, NULL); -} - -static VOID -FreeNonRecursiveMutex(PNRMUTEX mutex) -{ - /* No in-use check */ - CloseHandle(mutex); -} - -static DWORD -EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) -{ - return WaitForSingleObjectEx(mutex, milliseconds, FALSE); -} - -static BOOL -LeaveNonRecursiveMutex(PNRMUTEX mutex) -{ - return ReleaseSemaphore(mutex, 1, NULL); -} -#endif /* _PY_USE_CV_LOCKS */ - unsigned long PyThread_get_thread_ident(void); #ifdef PY_HAVE_THREAD_NATIVE_ID