From ec5c9f02b1554391d9db478e60cabbf23ca4b66e Mon Sep 17 00:00:00 2001 From: Michal Karol Date: Sun, 19 Jul 2026 13:21:32 +0200 Subject: [PATCH 1/6] Recreate the headers from raw form when part of signed data --- Lib/email/_policybase.py | 4 ++-- Lib/email/feedparser.py | 12 +++++++---- Lib/email/generator.py | 26 ++++++++++++------------ Lib/email/header.py | 9 +++++++++ Lib/email/message.py | 13 +++++++++--- Lib/email/policy.py | 4 ++-- Lib/test/test_email/data/msg_48.txt | 31 +++++++++++++++++++++++++++++ Lib/test/test_email/test_email.py | 28 ++++++++++++++++++++++++++ 8 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 Lib/test/test_email/data/msg_48.txt diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index e23843df44881f..2edef5b91b6cec 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -8,6 +8,7 @@ from email import header from email import charset as _charset from email.utils import _has_surrogates +from email.header import _HeaderValueWithRaw __all__ = [ 'Policy', @@ -316,8 +317,7 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) - value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') - return (name, value.rstrip('\r\n')) + return (name, _HeaderValueWithRaw(''.join((value, *sourcelines[1:])))) def header_store_parse(self, name, value): """+ diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index ae8ef32792b3e9..239df87c07c0b2 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -27,6 +27,7 @@ from email._policybase import compat32 from collections import deque from io import StringIO +from itertools import repeat, chain NLCRE = re.compile(r'\r\n|\r|\n') NLCRE_bol = re.compile(r'(\r\n|\r|\n)') @@ -215,9 +216,10 @@ def _pop_message(self): self._cur = None return retval - def _parsegen(self): + def _parsegen(self, is_signed_data: bool = False): # Create a new message and start by parsing headers. self._new_message() + self._cur.set_is_signed_data(is_signed_data) headers = [] # Collect the headers, searching for a line that doesn't match the RFC # 2822 header or continuation pattern (including an empty line). @@ -261,7 +263,7 @@ def _parsegen(self): # nested messages. A blank line separates the subparts. while True: self._input.push_eof_matcher(NLCRE.match) - for retval in self._parsegen(): + for retval in self._parsegen(is_signed_data): if retval is NeedMoreData: yield NeedMoreData continue @@ -295,7 +297,7 @@ def _parsegen(self): if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is # another RFC 5322 message. - for retval in self._parsegen(): + for retval in self._parsegen(is_signed_data): if retval is NeedMoreData: yield NeedMoreData continue @@ -303,6 +305,7 @@ def _parsegen(self): self._pop_message() return if self._cur.get_content_maintype() == 'multipart': + is_subpart_signed_generator = chain([True], repeat(is_signed_data)) if self._cur.get_content_subtype() == "signed" else repeat(is_signed_data) boundary = self._cur.get_boundary() if boundary is None: # The message /claims/ to be a multipart but it has not @@ -383,7 +386,8 @@ def boundarymatch(line): # Recurse to parse this subpart; the input stream points # at the subpart's first line. self._input.push_eof_matcher(boundarymatch) - for retval in self._parsegen(): + # Subpart is signed data if it is descendant of 'multipart/signed' message. + for retval in self._parsegen(next(is_subpart_signed_generator)): if retval is NeedMoreData: yield NeedMoreData continue diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ba11d63fba600a..ecfa68f36efdfd 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -15,6 +15,7 @@ from io import StringIO, BytesIO from email.utils import _has_surrogates from email.errors import HeaderWriteError +from email.header import _HeaderValueWithRaw UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. @@ -225,7 +226,12 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - folded = self.policy.fold(h, v) + # For messages that are signed, the message headers are reconstructed + # from the raw form to conform to the opaque requirement from RFC3156 3. + if msg.is_signed_data() and isinstance(v, _HeaderValueWithRaw): + folded = f"{h}:{v._raw_value}" + else: + folded = self.policy.fold(h, v) if self.policy.verify_generated_headers: linesep = self.policy.linesep if not folded.endswith(linesep): @@ -324,17 +330,6 @@ def _handle_multipart(self, msg): epilogue = msg.epilogue self._write_lines(epilogue) - def _handle_multipart_signed(self, msg): - # The contents of signed parts has to stay unmodified in order to keep - # the signature intact per RFC1847 2.1, so we disable header wrapping. - # RDM: This isn't enough to completely preserve the part, but it helps. - p = self.policy - self.policy = p.clone(max_line_length=0) - try: - self._handle_multipart(msg) - finally: - self.policy = p - def _handle_message_delivery_status(self, msg): # We can't just write the headers directly to self's file object # because this will leave an extra newline between the last header @@ -430,7 +425,12 @@ def _write_headers(self, msg): # This is almost the same as the string version, except for handling # strings with 8bit bytes. for h, v in msg.raw_items(): - folded = self.policy.fold_binary(h, v) + # For messages that are signed, the message headers are reconstructed + # from the raw form to conform to the opaque requirement from RFC3156 3. + if msg.is_signed_data() and isinstance(v, _HeaderValueWithRaw): + folded = self._encode(f"{h}:{v._raw_value}") + else: + folded = self.policy.fold_binary(h, v) if self.policy.verify_generated_headers: linesep = self.policy.linesep.encode() if not folded.endswith(linesep): diff --git a/Lib/email/header.py b/Lib/email/header.py index 220a84a7454b21..1b0c349197497d 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -580,3 +580,12 @@ def is_onlyws(self): def part_count(self): return super().__len__() + +class _HeaderValueWithRaw(str): + _raw_value: str + + def __new__(cls, value): + useful_value = value.lstrip(' \t\r\n').rstrip('\r\n') + instance = super().__new__(cls, useful_value) + instance._raw_value = value + return instance diff --git a/Lib/email/message.py b/Lib/email/message.py index 641fb2e944d431..a9de92b86d1e0e 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -164,6 +164,7 @@ def __init__(self, policy=compat32): self.defects = [] # Default content type self._default_type = 'text/plain' + self._is_signed_data = False def __str__(self): """Return the entire formatted message as a string. @@ -188,9 +189,9 @@ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): policy = self.policy if policy is None else policy fp = StringIO() g = Generator(fp, - mangle_from_=False, - maxheaderlen=maxheaderlen, - policy=policy) + mangle_from_=False, + maxheaderlen=maxheaderlen, + policy=policy) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -534,6 +535,12 @@ def raw_items(self): """ return iter(self._headers.copy()) + def set_is_signed_data(self, is_signed_data): + self._is_signed_data = is_signed_data + + def is_signed_data(self): + return self._is_signed_data + # # Additional useful stuff # diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 4169150101a29d..8c4813cb4d0fdd 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -12,6 +12,7 @@ validate_header_name ) from email.utils import _has_surrogates +from email.header import _HeaderValueWithRaw from email.headerregistry import HeaderRegistry as HeaderRegistry from email.contentmanager import raw_data_manager from email.message import EmailMessage @@ -131,8 +132,7 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) - value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') - return (name, value.rstrip('\r\n')) + return (name, _HeaderValueWithRaw(''.join((value, *sourcelines[1:])))) def header_store_parse(self, name, value): """+ diff --git a/Lib/test/test_email/data/msg_48.txt b/Lib/test/test_email/data/msg_48.txt new file mode 100644 index 00000000000000..7a1129dacb1715 --- /dev/null +++ b/Lib/test/test_email/data/msg_48.txt @@ -0,0 +1,31 @@ +From: +To: +Subject: test +In-Reply-To: somelonglonglonglonglongunwrappablemessageidtoreplyto@mail.foo.bar> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="borderline"; + protocol="application/pgp-signature"; micalg=pgp-sha1 + +This is an OpenPGP/MIME signed message (RFC 2440 and 3156) +--borderline +Content-Type: text/plain +In-Reply-To: + +MIME-Version: 1.0 + +This is the signed contents. + +--borderline +Content-Type: application/pgp-signature; name="signature.asc" +Content-Description: OpenPGP digital signature +Content-Disposition: attachment; filename="signature.asc" + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.0.6 (GNU/Linux) + +iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj +FGwfnRHFBFO/S4/DKysm0lI= +=t7+s +-----END PGP SIGNATURE----- + +--borderline-- diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index e40c82bba9af42..d0bbc6ddd5109d 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -2953,6 +2953,10 @@ def test_message_signed_idempotent(self): msg, text = self._msgobj('msg_45.txt') self._idempotent(msg, text) + def test_prewrapped_header_message_signed_idempotent(self): + msg, text = self._msgobj('msg_48.txt') + self._idempotent(msg, text) + def test_content_type(self): eq = self.assertEqual # Get a message object and reset the seek pointer for other tests @@ -5977,6 +5981,30 @@ def test_long_headers_flatten(self): result = fp.getvalue() self._signed_parts_eq(original, result) + def test_long_prewrapped_headers_as_string(self): + original, msg = self._msg_and_obj('msg_48.txt') + result = msg.as_string() + self._signed_parts_eq(original, result) + + def test_long_prewrapped_headers_as_string_maxheaderlen(self): + original, msg = self._msg_and_obj('msg_48.txt') + result = msg.as_string(maxheaderlen=60) + self._signed_parts_eq(original, result) + + def test_long_prewrapped_headers_flatten(self): + original, msg = self._msg_and_obj('msg_48.txt') + fp = StringIO() + Generator(fp).flatten(msg) + result = fp.getvalue() + self._signed_parts_eq(original, result) + + def test_if_parts_are_correctly_marked_as_signed_data(self): + msg = self._msgobj('msg_48.txt') + self.assertFalse(msg.is_signed_data()) + self.assertTrue(msg.get_payload(0).is_signed_data()) + self.assertFalse(msg.get_payload(1).is_signed_data()) + + class TestHeaderRegistry(TestEmailBase): # See issue gh-93010. def test_HeaderRegistry(self): From 0b78eb503c3fa7edee9f5e4f1b27b4293291d801 Mon Sep 17 00:00:00 2001 From: Michal Karol Date: Sun, 19 Jul 2026 13:25:33 +0200 Subject: [PATCH 2/6] Remove unnecessary formatting changes --- Lib/email/message.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/email/message.py b/Lib/email/message.py index a9de92b86d1e0e..7dcc416ef7c6f6 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -189,9 +189,9 @@ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): policy = self.policy if policy is None else policy fp = StringIO() g = Generator(fp, - mangle_from_=False, - maxheaderlen=maxheaderlen, - policy=policy) + mangle_from_=False, + maxheaderlen=maxheaderlen, + policy=policy) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() From bb15f51e2e6ea3d73e75b9212e53fb71702e8e8f Mon Sep 17 00:00:00 2001 From: Michal Karol Date: Sun, 19 Jul 2026 14:39:55 +0200 Subject: [PATCH 3/6] Fix typo in test message file --- Lib/test/test_email/data/msg_48.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_email/data/msg_48.txt b/Lib/test/test_email/data/msg_48.txt index 7a1129dacb1715..05786c881119e3 100644 --- a/Lib/test/test_email/data/msg_48.txt +++ b/Lib/test/test_email/data/msg_48.txt @@ -1,7 +1,7 @@ From: To: Subject: test -In-Reply-To: somelonglonglonglonglongunwrappablemessageidtoreplyto@mail.foo.bar> +In-Reply-To: MIME-Version: 1.0 Content-Type: multipart/signed; boundary="borderline"; protocol="application/pgp-signature"; micalg=pgp-sha1 From 71748c83b89e95bea33e49edf508aab8352224ec Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:24:37 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst diff --git a/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst new file mode 100644 index 00000000000000..da73c40c418e15 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst @@ -0,0 +1,2 @@ +Introduced structure for keeping raw header with whitespaces, to properly recreate headers for payloads that are PGP signed. +Raw header value overrides the default header folding of :meth:`email.policy.EmailPolicy.fold` and :meth:`email.policy.Compat32.fold` in :meth:`email.generator.Generator._write_headers` and bytes equivalents, preserving message and allowing for successfull signature validation. Contributed by Michał Karol. From 9289616b9ea4155ed4f4ed11139667cf06bebe0e Mon Sep 17 00:00:00 2001 From: Michal Karol Date: Sun, 19 Jul 2026 16:28:01 +0200 Subject: [PATCH 5/6] Fix whitespace lint issue after adding NEWS --- .../next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst index da73c40c418e15..f9ed26ec86c3db 100644 --- a/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst +++ b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst @@ -1,2 +1,2 @@ -Introduced structure for keeping raw header with whitespaces, to properly recreate headers for payloads that are PGP signed. +Introduced structure for keeping raw header with whitespaces, to properly recreate headers for payloads that are PGP signed. Raw header value overrides the default header folding of :meth:`email.policy.EmailPolicy.fold` and :meth:`email.policy.Compat32.fold` in :meth:`email.generator.Generator._write_headers` and bytes equivalents, preserving message and allowing for successfull signature validation. Contributed by Michał Karol. From d2f87628a590b9cd2fc9033130ae34f3821a32ed Mon Sep 17 00:00:00 2001 From: Michal Karol Date: Sun, 19 Jul 2026 19:38:48 +0200 Subject: [PATCH 6/6] Fix missing method reference in NEWS --- .../Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst index f9ed26ec86c3db..8bee6b0b58378d 100644 --- a/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst +++ b/Misc/NEWS.d/next/Library/2026-07-19-13-24-36.gh-issue-153822.5hQnrQ.rst @@ -1,2 +1,4 @@ Introduced structure for keeping raw header with whitespaces, to properly recreate headers for payloads that are PGP signed. -Raw header value overrides the default header folding of :meth:`email.policy.EmailPolicy.fold` and :meth:`email.policy.Compat32.fold` in :meth:`email.generator.Generator._write_headers` and bytes equivalents, preserving message and allowing for successfull signature validation. Contributed by Michał Karol. +Raw header value overrides the default header folding of :meth:`email.policy.EmailPolicy.fold` and :meth:`email.policy.Compat32.fold` +in :meth:`email.generator.Generator.flatten` and bytes equivalents, preserving message and allowing for successfull signature validation. +Contributed by Michał Karol.