diff --git a/ftfy/badness.py b/ftfy/badness.py index 38ec1f4..47e56f7 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -365,6 +365,13 @@ ^[ÃÂ][ ] | + # Windows-1252 mojibake of three-byte UTF-8 sequences in the U+2000 + # punctuation and symbol blocks, whose lead byte 0xE2 shows up as 'â'. + # This is restricted to 'â' on purpose: the same shape with other + # accented letters is ambiguous, e.g. 'Brontë…”' is not mojibake. + â[{common}][{start_punctuation}{end_punctuation}{currency}{numeric}{common}] + | + # Cases where  precedes a character as an encoding of exactly the same # character, and the character is common enough [a-z.,?!{end_punctuation}]  [ {start_punctuation}{end_punctuation}] diff --git a/tests/test-cases/synthetic.json b/tests/test-cases/synthetic.json index a939311..b232be7 100644 --- a/tests/test-cases/synthetic.json +++ b/tests/test-cases/synthetic.json @@ -204,5 +204,12 @@ "original": "OÙ ET QUAND?", "fixed": "OÙ ET QUAND?", "expect": "pass" + }, + { + "label": "Synthetic: vulgar fractions in Windows-1252 mojibake", + "comment": "issue #233. The lead byte of U+2153 is 0xE2, which shows up as 'â'; the second byte lands in the 'common' category, which on its own is not evidence of mojibake.", + "original": "â…“ and â…›", + "fixed": "⅓ and ⅛", + "expect": "pass" } ] \ No newline at end of file