From d7a3b7b4abaebacbbf8ca81f1952b56f3a631fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Lucas?= Date: Fri, 7 Aug 2026 22:03:54 -0300 Subject: [PATCH] Detect Windows-1252 mojibake of U+2000-block symbols (fixes #233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix_encoding("â…“") returned the input unchanged instead of "⅓". The bytes are E2 85 93, the UTF-8 encoding of U+2153, read as Windows-1252. The badness heuristic misses it because the second byte lands in the "common" category, which is deliberately not treated as evidence on its own. The obvious general rule -- [accented] [common] [punctuation] -- cannot be used: it also matches "Charlotte Brontë…”", which the test suite already guards against with a named negative case, and would turn it into a Korean syllable. The difference is the lead byte: 'â' is 0xE2, which opens the punctuation and symbol blocks, while 'ë' is 0xEB, which opens Hangul. So the new alternative is restricted to 'â', in the same spirit as the existing rule that is restricted to [ÂÃÎÐ] rather than to a whole category. --- ftfy/badness.py | 7 +++++++ tests/test-cases/synthetic.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/ftfy/badness.py b/ftfy/badness.py index 38ec1f4..47e56f7 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -365,6 +365,13 @@ ^[ÃÂ][ ] | + # Windows-1252 mojibake of three-byte UTF-8 sequences in the U+2000 + # punctuation and symbol blocks, whose lead byte 0xE2 shows up as 'â'. + # This is restricted to 'â' on purpose: the same shape with other + # accented letters is ambiguous, e.g. 'Brontë…”' is not mojibake. + â[{common}][{start_punctuation}{end_punctuation}{currency}{numeric}{common}] + | + # Cases where  precedes a character as an encoding of exactly the same # character, and the character is common enough [a-z.,?!{end_punctuation}]  [ {start_punctuation}{end_punctuation}] diff --git a/tests/test-cases/synthetic.json b/tests/test-cases/synthetic.json index a939311..b232be7 100644 --- a/tests/test-cases/synthetic.json +++ b/tests/test-cases/synthetic.json @@ -204,5 +204,12 @@ "original": "OÙ ET QUAND?", "fixed": "OÙ ET QUAND?", "expect": "pass" + }, + { + "label": "Synthetic: vulgar fractions in Windows-1252 mojibake", + "comment": "issue #233. The lead byte of U+2153 is 0xE2, which shows up as 'â'; the second byte lands in the 'common' category, which on its own is not evidence of mojibake.", + "original": "â…“ and â…›", + "fixed": "⅓ and ⅛", + "expect": "pass" } ] \ No newline at end of file