From 0bbb0f7b299673e5af9f5a95979896d23192bef8 Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 26 Aug 2026 08:05:01 -0700 Subject: [PATCH 1/4] Inline the per-character write in the CJK decoders Every CJK decoder emits output through OUTCHAR(), which called the exported _PyUnicodeWriter_WriteChar(). The codecs are shared extensions, so that is a cross-module call for each output character, and the function is nothing but the _PyUnicodeWriter_Prepare() macro plus a store that the caller could have done itself. OUTCHAR2, directly below it, was already written inline for exactly this reason. Use _PyUnicodeWriter_WriteCharInline() instead. One macro, and it applies to every codec in the package. | benchmark | before | after | change | spread | |-------------------------|----------:|----------:|-------:|-------:| | big5 decode 1.8MB | 6.32 ms | 2.56 ms | -59% | 0.3pp | | euc_kr decode 1.8MB | 6.94 ms | 3.18 ms | -54% | 0.1pp | | shift_jis decode 1.8MB | 6.93 ms | 3.17 ms | -54% | 0.4pp | | euc_jp decode 1.8MB | 6.32 ms | 3.18 ms | -50% | 0.2pp | | gb2312 decode 1.8MB | 6.32 ms | 3.18 ms | -50% | 0.3pp | | euc_jp decode kana | 3.26 ms | 2.05 ms | -37% | 4.0pp | | iso2022_jp decode | 9.48 ms | 7.09 ms | -25% | 8.3pp | Through the StreamReader wrappers, where Python-level call overhead dilutes it: | benchmark | before | after | change | |----------------------------|----------:|----------:|-------:| | euc_jp read() [1 call] | 6.33 ms | 3.18 ms | -50% | | euc_jp read(1024) xN | 7.06 ms | 3.93 ms | -44% | | euc_jp readlines() | 8.76 ms | 5.60 ms | -36% | | shift_jis readline() x20k | 11.91 ms | 8.25 ms | -30% | | euc_jp readline() x20k | 11.35 ms | 9.27 ms | -18% | Decoding every 1- and 2-byte sequence (65536 + 256 inputs) under each of the 23 CJK codecs, plus 3-byte probes, hashes identically before and after. Measured with the two builds in separate worktrees. These codecs live in shared .so files, so copying the python executable within one build tree does not isolate them: both copies load whichever module was built last, and the change measures as an exact no-op. --- Modules/cjkcodecs/cjkcodecs.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 9d86396f73b2b55..d85ab85f83c4c95 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -153,10 +153,13 @@ get_module_state(PyObject *mod) #define INCHAR1 (PyUnicode_READ(kind, data, *inpos)) #define INCHAR2 (PyUnicode_READ(kind, data, *inpos + 1)) +/* Decoders emit one character at a time, so this must stay inline: the + out-of-line _PyUnicodeWriter_WriteChar() is a cross-module call per output + character. OUTCHAR2 below is written inline for the same reason. */ #define OUTCHAR(c) \ do { \ - if (_PyUnicodeWriter_WriteChar(writer, (c)) < 0) \ - return MBERR_EXCEPTION; \ + if (_PyUnicodeWriter_WriteCharInline(writer, (c)) < 0) \ + return MBERR_EXCEPTION; \ } while (0) #define OUTCHAR2(c1, c2) \ From 6a35e934372829df68bb789c59270ed2cedb868f Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:08:59 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst b/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst new file mode 100644 index 000000000000000..1bd8b29732fb95c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst @@ -0,0 +1 @@ +Optimize codecs.decode() calls From 94dc90cc84ae51a0f52f390b22a3d42cb4d5c9ab Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Fri, 28 Aug 2026 14:09:22 -0700 Subject: [PATCH 3/4] The comment is probably not needed --- Modules/cjkcodecs/cjkcodecs.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index d85ab85f83c4c95..41e1287c8650701 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -153,9 +153,6 @@ get_module_state(PyObject *mod) #define INCHAR1 (PyUnicode_READ(kind, data, *inpos)) #define INCHAR2 (PyUnicode_READ(kind, data, *inpos + 1)) -/* Decoders emit one character at a time, so this must stay inline: the - out-of-line _PyUnicodeWriter_WriteChar() is a cross-module call per output - character. OUTCHAR2 below is written inline for the same reason. */ #define OUTCHAR(c) \ do { \ if (_PyUnicodeWriter_WriteCharInline(writer, (c)) < 0) \ From 0e120a81c659713eca7a5898c6a491c5f6b896c1 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 29 Aug 2026 12:04:50 +0900 Subject: [PATCH 4/4] Delete Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst --- .../next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst b/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst deleted file mode 100644 index 1bd8b29732fb95c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-28-21-08-58.gh-issue-156537.aJhAM2.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize codecs.decode() calls