diff --git a/ftfy/badness.py b/ftfy/badness.py index 38ec1f4..813ef89 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -301,7 +301,7 @@ | [{box}] [{kaomoji}] | - [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}{law}] [{box}] + [{lower_accented}{upper_accented}{lower_common}{upper_common}{currency}{numeric}{start_punctuation}{end_punctuation}{law}] [{box}] | [{box}] [{end_punctuation}] | diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 43d117c..f5af37e 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -23,6 +23,7 @@ "iso-8859-2", "macroman", "cp437", + "koi8-r", ] SINGLE_QUOTE_RE = re.compile("[\u02bc\u2018-\u201b]") diff --git a/tests/test-cases/in-the-wild.json b/tests/test-cases/in-the-wild.json index b40c838..c32b11e 100644 --- a/tests/test-cases/in-the-wild.json +++ b/tests/test-cases/in-the-wild.json @@ -447,5 +447,12 @@ "fixed-encoding": "Ningbo,China", "fixed": "Ningbo,China", "expect": "pass" + }, + { + "label": "Russian text in KOI8-R mojibake", + "comment": "issue #231. Reported as CP437/CP866, but it is KOI8-R: 0xD0 and 0xD1 decode to 'п' and 'я', which are the UTF-8 lead bytes of Cyrillic.", + "original": "п∙я│п╩п╦ п▓я▀ п╫п╣ я▐п╡п╩я▐п╣я┌п╣я│я▄ п╟п╢я─п╣я│п╟я┌п╬п╪", + "fixed": "Если Вы не являетесь адресатом", + "expect": "pass" } ] \ No newline at end of file