From 39955446b5ae398a9c865381ad88ad399e918f01 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 28 Aug 2026 15:54:54 +0300 Subject: [PATCH] gh-156513: Match a backreference case-insensitively as a literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sre_lower_unicode() now returns the lowercase of the simple case folding of the character, computed by the new _PyUnicode_ToFoldedLowercase(), with the four pairs which no case mapping unifies hardcoded. The key is the same for all characters matched case-insensitively, so a backreference matches what a literal matches: (?i)(.)\1 matches 'ςσ' as (?i)ς already matched 'σ'. _casefix._EXTRA_CASES is no longer needed and is removed, together with the alternatives which the compiler expanded a literal into. Its generator now emits no table, but keeps computing the groups of characters which have to be matched case-insensitively, and fails if sre_lower_unicode() does not fold such a group to a single code. Co-Authored-By: Claude Opus 5 (1M context) --- Include/internal/pycore_unicodectype.h | 1 + Lib/re/_casefix.py | 109 +----------------- Lib/re/_compiler.py | 30 ++--- Lib/re/_optimizer.py | 27 ++--- Lib/test/test_re.py | 67 ++++++++++- ...-08-28-12-32-03.gh-issue-156513.hD62em.rst | 3 + Modules/_sre/sre.c | 17 ++- Modules/_sre/sre_lib.h | 2 +- Objects/unicodectype.c | 23 ++++ Tools/build/generate_re_casefix.py | 72 ++++-------- 10 files changed, 151 insertions(+), 200 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-28-12-32-03.gh-issue-156513.hD62em.rst diff --git a/Include/internal/pycore_unicodectype.h b/Include/internal/pycore_unicodectype.h index 523bdb56b09cde4..300cc90ea3bf894 100644 --- a/Include/internal/pycore_unicodectype.h +++ b/Include/internal/pycore_unicodectype.h @@ -12,6 +12,7 @@ extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); +extern Py_UCS4 _PyUnicode_ToFoldedLowercase(Py_UCS4 ch); extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); extern int _PyUnicode_IsCased(Py_UCS4 ch); diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index fed2d84fc01473c..412f03d8a0d4352 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,106 +1,5 @@ # Auto-generated by Tools/build/generate_re_casefix.py. - -# Maps the code of lowercased character to codes of different lowercased -# characters which have the same uppercase. -_EXTRA_CASES = { - # LATIN SMALL LETTER I: LATIN SMALL LETTER DOTLESS I - 0x0069: (0x0131,), # 'i': 'ı' - # LATIN SMALL LETTER S: LATIN SMALL LETTER LONG S - 0x0073: (0x017f,), # 's': 'ſ' - # MICRO SIGN: GREEK SMALL LETTER MU - 0x00b5: (0x03bc,), # 'µ': 'μ' - # LATIN SMALL LETTER DOTLESS I: LATIN SMALL LETTER I - 0x0131: (0x0069,), # 'ı': 'i' - # LATIN SMALL LETTER LONG S: LATIN SMALL LETTER S - 0x017f: (0x0073,), # 'ſ': 's' - # COMBINING GREEK YPOGEGRAMMENI: GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI - 0x0345: (0x03b9, 0x1fbe), # '\u0345': 'ιι' - # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA - 0x0390: (0x1fd3,), # 'ΐ': 'ΐ' - # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA - 0x03b0: (0x1fe3,), # 'ΰ': 'ΰ' - # GREEK SMALL LETTER BETA: GREEK BETA SYMBOL - 0x03b2: (0x03d0,), # 'β': 'ϐ' - # GREEK SMALL LETTER EPSILON: GREEK LUNATE EPSILON SYMBOL - 0x03b5: (0x03f5,), # 'ε': 'ϵ' - # GREEK SMALL LETTER THETA: GREEK THETA SYMBOL - 0x03b8: (0x03d1,), # 'θ': 'ϑ' - # GREEK SMALL LETTER IOTA: COMBINING GREEK YPOGEGRAMMENI, GREEK PROSGEGRAMMENI - 0x03b9: (0x0345, 0x1fbe), # 'ι': '\u0345ι' - # GREEK SMALL LETTER KAPPA: GREEK KAPPA SYMBOL - 0x03ba: (0x03f0,), # 'κ': 'ϰ' - # GREEK SMALL LETTER MU: MICRO SIGN - 0x03bc: (0x00b5,), # 'μ': 'µ' - # GREEK SMALL LETTER PI: GREEK PI SYMBOL - 0x03c0: (0x03d6,), # 'π': 'ϖ' - # GREEK SMALL LETTER RHO: GREEK RHO SYMBOL - 0x03c1: (0x03f1,), # 'ρ': 'ϱ' - # GREEK SMALL LETTER FINAL SIGMA: GREEK SMALL LETTER SIGMA - 0x03c2: (0x03c3,), # 'ς': 'σ' - # GREEK SMALL LETTER SIGMA: GREEK SMALL LETTER FINAL SIGMA - 0x03c3: (0x03c2,), # 'σ': 'ς' - # GREEK SMALL LETTER PHI: GREEK PHI SYMBOL - 0x03c6: (0x03d5,), # 'φ': 'ϕ' - # GREEK BETA SYMBOL: GREEK SMALL LETTER BETA - 0x03d0: (0x03b2,), # 'ϐ': 'β' - # GREEK THETA SYMBOL: GREEK SMALL LETTER THETA - 0x03d1: (0x03b8,), # 'ϑ': 'θ' - # GREEK PHI SYMBOL: GREEK SMALL LETTER PHI - 0x03d5: (0x03c6,), # 'ϕ': 'φ' - # GREEK PI SYMBOL: GREEK SMALL LETTER PI - 0x03d6: (0x03c0,), # 'ϖ': 'π' - # GREEK KAPPA SYMBOL: GREEK SMALL LETTER KAPPA - 0x03f0: (0x03ba,), # 'ϰ': 'κ' - # GREEK RHO SYMBOL: GREEK SMALL LETTER RHO - 0x03f1: (0x03c1,), # 'ϱ': 'ρ' - # GREEK LUNATE EPSILON SYMBOL: GREEK SMALL LETTER EPSILON - 0x03f5: (0x03b5,), # 'ϵ': 'ε' - # CYRILLIC SMALL LETTER VE: CYRILLIC SMALL LETTER ROUNDED VE - 0x0432: (0x1c80,), # 'в': 'ᲀ' - # CYRILLIC SMALL LETTER DE: CYRILLIC SMALL LETTER LONG-LEGGED DE - 0x0434: (0x1c81,), # 'д': 'ᲁ' - # CYRILLIC SMALL LETTER O: CYRILLIC SMALL LETTER NARROW O - 0x043e: (0x1c82,), # 'о': 'ᲂ' - # CYRILLIC SMALL LETTER ES: CYRILLIC SMALL LETTER WIDE ES - 0x0441: (0x1c83,), # 'с': 'ᲃ' - # CYRILLIC SMALL LETTER TE: CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE - 0x0442: (0x1c84, 0x1c85), # 'т': 'ᲄᲅ' - # CYRILLIC SMALL LETTER HARD SIGN: CYRILLIC SMALL LETTER TALL HARD SIGN - 0x044a: (0x1c86,), # 'ъ': 'ᲆ' - # CYRILLIC SMALL LETTER YAT: CYRILLIC SMALL LETTER TALL YAT - 0x0463: (0x1c87,), # 'ѣ': 'ᲇ' - # CYRILLIC SMALL LETTER ROUNDED VE: CYRILLIC SMALL LETTER VE - 0x1c80: (0x0432,), # 'ᲀ': 'в' - # CYRILLIC SMALL LETTER LONG-LEGGED DE: CYRILLIC SMALL LETTER DE - 0x1c81: (0x0434,), # 'ᲁ': 'д' - # CYRILLIC SMALL LETTER NARROW O: CYRILLIC SMALL LETTER O - 0x1c82: (0x043e,), # 'ᲂ': 'о' - # CYRILLIC SMALL LETTER WIDE ES: CYRILLIC SMALL LETTER ES - 0x1c83: (0x0441,), # 'ᲃ': 'с' - # CYRILLIC SMALL LETTER TALL TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER THREE-LEGGED TE - 0x1c84: (0x0442, 0x1c85), # 'ᲄ': 'тᲅ' - # CYRILLIC SMALL LETTER THREE-LEGGED TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE - 0x1c85: (0x0442, 0x1c84), # 'ᲅ': 'тᲄ' - # CYRILLIC SMALL LETTER TALL HARD SIGN: CYRILLIC SMALL LETTER HARD SIGN - 0x1c86: (0x044a,), # 'ᲆ': 'ъ' - # CYRILLIC SMALL LETTER TALL YAT: CYRILLIC SMALL LETTER YAT - 0x1c87: (0x0463,), # 'ᲇ': 'ѣ' - # CYRILLIC SMALL LETTER UNBLENDED UK: CYRILLIC SMALL LETTER MONOGRAPH UK - 0x1c88: (0xa64b,), # 'ᲈ': 'ꙋ' - # LATIN SMALL LETTER S WITH DOT ABOVE: LATIN SMALL LETTER LONG S WITH DOT ABOVE - 0x1e61: (0x1e9b,), # 'ṡ': 'ẛ' - # LATIN SMALL LETTER LONG S WITH DOT ABOVE: LATIN SMALL LETTER S WITH DOT ABOVE - 0x1e9b: (0x1e61,), # 'ẛ': 'ṡ' - # GREEK PROSGEGRAMMENI: COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA - 0x1fbe: (0x0345, 0x03b9), # 'ι': '\u0345ι' - # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS - 0x1fd3: (0x0390,), # 'ΐ': 'ΐ' - # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS - 0x1fe3: (0x03b0,), # 'ΰ': 'ΰ' - # CYRILLIC SMALL LETTER MONOGRAPH UK: CYRILLIC SMALL LETTER UNBLENDED UK - 0xa64b: (0x1c88,), # 'ꙋ': 'ᲈ' - # LATIN SMALL LIGATURE LONG S T: LATIN SMALL LIGATURE ST - 0xfb05: (0xfb06,), # 'ſt': 'st' - # LATIN SMALL LIGATURE ST: LATIN SMALL LIGATURE LONG S T - 0xfb06: (0xfb05,), # 'st': 'ſt' -} +# +# Every character which has to be matched case-insensitively with another one +# is folded to the same code by sre_lower_unicode(), so this module has no +# tables; the script checks that this holds. diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 58a24964c3b374f..dbdd8217a19800f 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -13,7 +13,6 @@ import _sre from . import _parser from ._constants import * -from ._casefix import _EXTRA_CASES from ._optimizer import ( _combine_flags, _compile_charset, _optimize_charset, _compile_info, _simple, _CHARSET_ALL, _CODEBITS, MAXCODE, optimize, @@ -41,12 +40,12 @@ def _compile(code, pattern, flags): ASSERT_CODES = _ASSERT_CODES iscased = None tolower = None - fixes = None + isunicode = False if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE: if flags & SRE_FLAG_UNICODE: iscased = _sre.unicode_iscased tolower = _sre.unicode_tolower - fixes = _EXTRA_CASES + isunicode = True else: iscased = _sre.ascii_iscased tolower = _sre.ascii_tolower @@ -62,25 +61,10 @@ def _compile(code, pattern, flags): emit(op) emit(av) else: - lo = tolower(av) - if not fixes: # ascii - emit(OP_IGNORE[op]) - emit(lo) - elif lo not in fixes: - emit(OP_UNICODE_IGNORE[op]) - emit(lo) - else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - if op is NOT_LITERAL: - emit(NEGATE) - for k in (lo,) + fixes[lo]: - emit(LITERAL) - emit(k) - emit(FAILURE) - code[skip] = _len(code) - skip + emit(OP_UNICODE_IGNORE[op] if isunicode else OP_IGNORE[op]) + emit(tolower(av)) elif op is IN: - charset, hascased = _optimize_charset(av, iscased, tolower, fixes) + charset, hascased = _optimize_charset(av, iscased, tolower, isunicode) if not charset: emit(FAILURE) elif charset == _CHARSET_ALL: @@ -90,7 +74,7 @@ def _compile(code, pattern, flags): emit(IN_LOC_IGNORE) elif not hascased: emit(IN) - elif not fixes: # ascii + elif not isunicode: emit(IN_IGNORE) else: emit(IN_UNI_IGNORE) @@ -192,7 +176,7 @@ def _compile(code, pattern, flags): emit(op) elif flags & SRE_FLAG_LOCALE: emit(GROUPREF_LOC_IGNORE) - elif not fixes: # ascii + elif not isunicode: emit(GROUPREF_IGNORE) else: emit(GROUPREF_UNI_IGNORE) diff --git a/Lib/re/_optimizer.py b/Lib/re/_optimizer.py index 6a0bb5a2973eaec..36b70f63727f03e 100644 --- a/Lib/re/_optimizer.py +++ b/Lib/re/_optimizer.py @@ -55,7 +55,7 @@ def _compile_charset(charset, flags, code): raise PatternError(f"internal: unsupported set operator {op!r}") emit(FAILURE) -def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): +def _optimize_charset(charset, iscased=None, fixup=None, isunicode=False): # internal: optimize character set. # # The engine's charset() walk toggles polarity on every NEGATE (see @@ -67,7 +67,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): if not negates or negates == [0]: # Fast path: a plain union, optionally complemented as a whole -- every # charset the parser produces today, optimized as before. - return _optimize_charset_segment(charset, iscased, fixup, fixes) + return _optimize_charset_segment(charset, iscased, fixup, isunicode) # Optimize each NEGATE-delimited run on its own. _allow_anyall is off: the # [\s\S] -> ANY_ALL / [^\s\S] -> empty shortcuts rewrite a whole set and @@ -78,7 +78,8 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): for i in negates + [len(charset)]: if i > start: # skip an empty run (e.g. a leading NEGATE) opt, cased = _optimize_charset_segment( - charset[start:i], iscased, fixup, fixes, _allow_anyall=False) + charset[start:i], iscased, fixup, isunicode, + _allow_anyall=False) out.extend(opt) hascased |= cased if i < len(charset): @@ -86,8 +87,8 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): start = i + 1 return out, hascased -def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None, - _allow_anyall=True): +def _optimize_charset_segment(charset, iscased=None, fixup=None, + isunicode=False, _allow_anyall=True): # internal: optimize one NEGATE-free union of character-set members out = [] tail = [] @@ -100,9 +101,6 @@ def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None, if fixup: # IGNORECASE and not LOCALE av = fixup(av) charmap[av] = 1 - if fixes and av in fixes: - for k in fixes[av]: - charmap[k] = 1 if not hascased and iscased(av): hascased = True else: @@ -110,15 +108,8 @@ def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None, elif op is RANGE: r = range(av[0], av[1]+1) if fixup: # IGNORECASE and not LOCALE - if fixes: - for i in map(fixup, r): - charmap[i] = 1 - if i in fixes: - for k in fixes[i]: - charmap[k] = 1 - else: - for i in map(fixup, r): - charmap[i] = 1 + for i in map(fixup, r): + charmap[i] = 1 if not hascased: hascased = any(map(iscased, r)) else: @@ -151,7 +142,7 @@ def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None, # Also, both c.lower() and c.lower().upper() are single # characters for every non-BMP character. if op is RANGE: - if fixes: # not ASCII + if isunicode: op = RANGE_UNI_IGNORE hascased = True else: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index ff106c1b3415668..61048f854c4ee17 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1327,6 +1327,46 @@ def test_ignore_case(self): self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) + # Cherokee letters case fold to their uppercase. Unicode 8.0+. + assert '\uab70'.casefold() == '\u13a0'.casefold() == '\u13a0' # 'ꭰ', 'Ꭰ' + self.assertTrue(re.match('\u13a0', '\uab70', re.I)) + self.assertTrue(re.match('\uab70', '\u13a0', re.I)) + + # Characters which the matcher maps to the same code (see + # _sre.unicode_tolower), and which are therefore matched + # case-insensitively wherever it compares characters. + FOLDED_PAIRS = [ + ('\u03c2', '\u03c3'), # 'ς', 'σ' + ('\u03c3', '\u03a3'), # 'σ', 'Σ' + ('\u00b5', '\u03bc'), # 'µ', 'μ' + ('\u017f', 's'), # 'ſ' + ('\u00df', '\u1e9e'), # 'ß', 'ẞ' + ('k', '\u212a'), # KELVIN SIGN + ('\u0432', '\u1c80'), # 'в', 'ᲀ' + ('\uab70', '\u13a0'), # 'ꭰ', 'Ꭰ' + # Not unified by the case folding (see sre_lower_unicode). + ('i', '\u0131'), # LATIN SMALL LETTER DOTLESS I + ('\u0390', '\u1fd3'), # 'ΐ' with tonos and with oxia + ('\u03b0', '\u1fe3'), # 'ΰ' with tonos and with oxia + ('\ufb05', '\ufb06'), # 'ſt', 'st' + ] + + def test_ignore_case_backreference(self): + for a, b in self.FOLDED_PAIRS: + with self.subTest(pair=(a, b)): + self.assertTrue(re.fullmatch(r'(.)\1', a + b, re.I)) + self.assertTrue(re.fullmatch(r'(.)\1', b + a, re.I)) + + def test_ignore_case_folded_pairs(self): + for a, b in self.FOLDED_PAIRS: + with self.subTest(pair=(a, b)): + self.assertTrue(re.match(a, b, re.I)) + self.assertTrue(re.match(b, a, re.I)) + self.assertTrue(re.match('[%s]' % a, b, re.I)) + self.assertTrue(re.match('[%s]' % b, a, re.I)) + self.assertIsNone(re.match('[^%s]' % a, b, re.I)) + self.assertIsNone(re.match('[^%s]' % b, a, re.I)) + def test_ignore_case_set(self): self.assertTrue(re.match(r'[19A]', 'A', re.I)) self.assertTrue(re.match(r'[19a]', 'a', re.I)) @@ -3196,11 +3236,18 @@ def test_case_helpers(self): self.assertEqual(_sre.ascii_iscased(i), iscased) self.assertEqual(_sre.unicode_iscased(i), iscased) + # Characters which the case folding does not unify. + fold_exceptions = {0x0131: 0x0069, 0x1fd3: 0x0390, + 0x1fe3: 0x03b0, 0xfb05: 0xfb06} for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: c = chr(i) self.assertEqual(_sre.ascii_tolower(i), i) - if i != 0x0130: - self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) + if i != 0x0130 and i not in fold_exceptions: + # unicode_tolower() is the case folding when it is a single + # character, and the lowercase otherwise. + f = c.casefold() + self.assertEqual(_sre.unicode_tolower(i), + ord(f) if len(f) == 1 else ord(c.lower())) iscased = c != c.lower() or c != c.upper() self.assertFalse(_sre.ascii_iscased(i)) self.assertEqual(_sre.unicode_iscased(i), @@ -3211,6 +3258,22 @@ def test_case_helpers(self): self.assertFalse(_sre.ascii_iscased(0x0130)) self.assertTrue(_sre.unicode_iscased(0x0130)) + # Case folding, unlike lowercasing, unifies these. + self.assertEqual(_sre.unicode_tolower(0x00b5), 0x03bc) # 'µ', 'μ' + self.assertEqual(_sre.unicode_tolower(0x017f), ord('s')) # 'ſ' + # The full case folding of both sharp S characters is "ss", so they + # are matched through their common lowercase. + self.assertEqual(_sre.unicode_tolower(0x00df), 0x00df) # 'ß' + self.assertEqual(_sre.unicode_tolower(0x1e9e), 0x00df) # 'ẞ' + # Cherokee letters case fold to their uppercase, which is lowercased + # back so that the two forms share a key. + self.assertEqual(_sre.unicode_tolower(0x13a0), 0xab70) # 'Ꭰ', 'ꭰ' + self.assertEqual(_sre.unicode_tolower(0xab70), 0xab70) + for i, lo in fold_exceptions.items(): + with self.subTest(char=chr(i)): + self.assertEqual(_sre.unicode_tolower(i), lo) + + @cpython_only def test_dealloc(self): # issue 3299: check for segfault in debug build diff --git a/Misc/NEWS.d/next/Library/2026-08-28-12-32-03.gh-issue-156513.hD62em.rst b/Misc/NEWS.d/next/Library/2026-08-28-12-32-03.gh-issue-156513.hD62em.rst new file mode 100644 index 000000000000000..ec0d795f471ade7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-28-12-32-03.gh-issue-156513.hD62em.rst @@ -0,0 +1,3 @@ +Fix case-insensitive matching of a backreference in :mod:`re`: it now matches +the characters which are matched as a literal, so that ``(?i)(.)\1`` matches +``'ςσ'``. diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 031e972aeba0312..414142fc5c55cd9 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -214,9 +214,24 @@ static unsigned int sre_upper_locale(unsigned int ch) #define SRE_UNI_IS_PRINT(ch) ((SRE_UNI_IS_GRAPH(ch) || SRE_UNI_IS_BLANK(ch)) && \ !SRE_IS_CC(ch)) +/* The key by which characters are compared case-insensitively: the lowercase + of the simple case folding. Folding, unlike lowercasing, makes MICRO SIGN + and GREEK SMALL LETTER MU compare equal; lowercasing the result keeps the + key below the case pairs, as Cherokee letters fold to their uppercase. + When the full folding is not a single character (as "ss" for both SHARP S + characters), the character is lowercased on its own. */ static unsigned int sre_lower_unicode(unsigned int ch) { - return (unsigned int) Py_UNICODE_TOLOWER(ch); + switch (ch) { + /* Characters which the case folding does not unify: they only have the + same uppercase, or the same full folding of more than one character. + All are Unicode 1.1. */ + case 0x0131: return 0x0069; /* DOTLESS I -> I */ + case 0x1fd3: return 0x0390; /* IOTA WITH DIALYTIKA AND OXIA -> TONOS */ + case 0x1fe3: return 0x03b0; /* UPSILON WITH DIALYTIKA AND OXIA -> TONOS */ + case 0xfb05: return 0xfb06; /* LIGATURE LONG S T -> LIGATURE ST */ + } + return (unsigned int) _PyUnicode_ToFoldedLowercase((Py_UCS4)ch); } static unsigned int sre_upper_unicode(unsigned int ch) diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index fbdbaefb399193d..44a104315465913 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -134,7 +134,7 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch) /* */ { SRE_CODE uch; - /* ch is already lower cased */ + /* ch is already case folded (see sre_lower_unicode) */ if (set[0] <= ch && ch <= set[1]) return ok; uch = sre_upper_unicode(ch); diff --git a/Objects/unicodectype.c b/Objects/unicodectype.c index fdd380190ac1ecc..95a629c7b5c7341 100644 --- a/Objects/unicodectype.c +++ b/Objects/unicodectype.c @@ -262,6 +262,29 @@ int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res) return _PyUnicode_ToLowerFull(ch, res); } +/* The lowercase of the simple case folding of ch: the full case folding when + that is a single character, else ch itself, lowercased in both cases. + Equivalent to _PyUnicode_ToLowercase() of the result of + _PyUnicode_ToFoldedFull() of length 1, but reads the type record once + except for the few characters having a special single-character + folding. */ +Py_UCS4 _PyUnicode_ToFoldedLowercase(Py_UCS4 ch) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + if (!(ctype->flags & EXTENDED_CASE_MASK)) { + return ch + ctype->lower; + } + if (((ctype->lower >> 20) & 7) == 1) { + /* A single-character folding. It can be an uppercase character -- + Cherokee letters fold to their uppercase -- so lowercase it. */ + int index = (ctype->lower & 0xFFFF) + (ctype->lower >> 24); + return _PyUnicode_ToLowercase(_PyUnicode_ExtendedCase[index]); + } + /* No folding, or a multi-character one: the simple lowercase of ch. */ + return _PyUnicode_ExtendedCase[ctype->lower & 0xFFFF]; +} + int _PyUnicode_IsCased(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); diff --git a/Tools/build/generate_re_casefix.py b/Tools/build/generate_re_casefix.py index 9345b8f4efe09d9..1d0e4a5d63cc5f8 100755 --- a/Tools/build/generate_re_casefix.py +++ b/Tools/build/generate_re_casefix.py @@ -20,12 +20,10 @@ def update_file(file, content): re_casefix_template = f"""\ # Auto-generated by {SCRIPT_NAME}. - -# Maps the code of lowercased character to codes of different lowercased -# characters which have the same uppercase. -_EXTRA_CASES = {{ -%s -}} +# +# Every character which has to be matched case-insensitively with another one +# is folded to the same code by sre_lower_unicode(), so this module has no +# tables; the script checks that this holds. """ def uname(i): @@ -41,55 +39,29 @@ def alpha(i): def main(outfile='Lib/re/_casefix.py'): - # Find sets of characters which have the same uppercase. + import _sre + + # Characters which case fold to the same string, or which have the same + # uppercase, are matched case-insensitively, so sre_lower_unicode() has to + # fold them to the same code -- by the case folding, or by one of the few + # exceptions which it hardcodes. equivalent_chars = collections.defaultdict(str) for c in map(chr, range(sys.maxunicode + 1)): - equivalent_chars[c.upper()] += c - equivalent_chars = [t for t in equivalent_chars.values() if len(t) > 1] - - # List of codes of lowercased characters which have the same uppercase. - equivalent_lower_codes = [sorted(t) - for s in equivalent_chars - for t in [{ord(c.lower()) for c in s}] - if len(t) > 1] - - bad_codes = [] - for t in equivalent_lower_codes: - for i in t: - if i > 0xffff: - bad_codes.extend(t) - try: - bad_codes.append(ord(chr(i).upper())) - except (ValueError, TypeError): - pass - break - if bad_codes: - print('Case-insensitive matching may not work correctly for character:', + equivalent_chars['f' + c.casefold()] += c + equivalent_chars['u' + c.upper()] += c + not_folded = sorted(t for s in equivalent_chars.values() if len(s) > 1 + for t in [sorted({_sre.unicode_tolower(ord(c)) + for c in s})] + if len(t) > 1) + if not_folded: + print('Case-insensitive matching does not work for characters:', file=sys.stderr) - for i in sorted(bad_codes): - print(" '%s' (U+%04x, %s)" % (alpha(i), i, uname(i)), - file=sys.stderr) + for t in not_folded: + print(' ' + ', '.join("'%s' (U+%04x, %s)" % (alpha(i), i, uname(i)) + for i in t), file=sys.stderr) sys.exit(1) - mapping = {i: tuple(j for j in t if i != j) - for t in equivalent_lower_codes - for i in t} - - items = [] - for i, t in sorted(mapping.items()): - items.append(' # %s: %s' % ( - uname(i), - ', '.join(map(uname, t)), - )) - items.append(" %r: %r, # '%s': '%s'" % ( - hexint(i), - tuple(map(hexint, t)), - alpha(i), - ''.join(map(alpha, t)), - )) - - update_file(outfile, re_casefix_template % '\n'.join(items)) - + update_file(outfile, re_casefix_template) if __name__ == '__main__': import sys