gh-156513: Match a backreference case-insensitively as a literal

Include/internal/pycore_unicodectype.h

@@ -12,6 +12,7 @@ extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res);
 extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res);
 extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res);
 extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res);
+extern Py_UCS4 _PyUnicode_ToFoldedLowercase(Py_UCS4 ch);
 extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch);
 extern int _PyUnicode_IsCased(Py_UCS4 ch);
 

Lib/re/_casefix.py

@@ -1,106 +1,5 @@
 # Auto-generated by Tools/build/generate_re_casefix.py.
-
-# Maps the code of lowercased character to codes of different lowercased
-# characters which have the same uppercase.
-_EXTRA_CASES = {
-    # LATIN SMALL LETTER I: LATIN SMALL LETTER DOTLESS I
-    0x0069: (0x0131,), # 'i': 'ı'
-    # LATIN SMALL LETTER S: LATIN SMALL LETTER LONG S
-    0x0073: (0x017f,), # 's': 'ſ'
-    # MICRO SIGN: GREEK SMALL LETTER MU
-    0x00b5: (0x03bc,), # 'µ': 'μ'
-    # LATIN SMALL LETTER DOTLESS I: LATIN SMALL LETTER I
-    0x0131: (0x0069,), # 'ı': 'i'
-    # LATIN SMALL LETTER LONG S: LATIN SMALL LETTER S
-    0x017f: (0x0073,), # 'ſ': 's'
-    # COMBINING GREEK YPOGEGRAMMENI: GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
-    0x0345: (0x03b9, 0x1fbe), # '\u0345': 'ιι'
-    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
-    0x0390: (0x1fd3,), # 'ΐ': 'ΐ'
-    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
-    0x03b0: (0x1fe3,), # 'ΰ': 'ΰ'
-    # GREEK SMALL LETTER BETA: GREEK BETA SYMBOL
-    0x03b2: (0x03d0,), # 'β': 'ϐ'
-    # GREEK SMALL LETTER EPSILON: GREEK LUNATE EPSILON SYMBOL
-    0x03b5: (0x03f5,), # 'ε': 'ϵ'
-    # GREEK SMALL LETTER THETA: GREEK THETA SYMBOL
-    0x03b8: (0x03d1,), # 'θ': 'ϑ'
-    # GREEK SMALL LETTER IOTA: COMBINING GREEK YPOGEGRAMMENI, GREEK PROSGEGRAMMENI
-    0x03b9: (0x0345, 0x1fbe), # 'ι': '\u0345ι'
-    # GREEK SMALL LETTER KAPPA: GREEK KAPPA SYMBOL
-    0x03ba: (0x03f0,), # 'κ': 'ϰ'
-    # GREEK SMALL LETTER MU: MICRO SIGN
-    0x03bc: (0x00b5,), # 'μ': 'µ'
-    # GREEK SMALL LETTER PI: GREEK PI SYMBOL
-    0x03c0: (0x03d6,), # 'π': 'ϖ'
-    # GREEK SMALL LETTER RHO: GREEK RHO SYMBOL
-    0x03c1: (0x03f1,), # 'ρ': 'ϱ'
-    # GREEK SMALL LETTER FINAL SIGMA: GREEK SMALL LETTER SIGMA
-    0x03c2: (0x03c3,), # 'ς': 'σ'
-    # GREEK SMALL LETTER SIGMA: GREEK SMALL LETTER FINAL SIGMA
-    0x03c3: (0x03c2,), # 'σ': 'ς'
-    # GREEK SMALL LETTER PHI: GREEK PHI SYMBOL
-    0x03c6: (0x03d5,), # 'φ': 'ϕ'
-    # GREEK BETA SYMBOL: GREEK SMALL LETTER BETA
-    0x03d0: (0x03b2,), # 'ϐ': 'β'
-    # GREEK THETA SYMBOL: GREEK SMALL LETTER THETA
-    0x03d1: (0x03b8,), # 'ϑ': 'θ'
-    # GREEK PHI SYMBOL: GREEK SMALL LETTER PHI
-    0x03d5: (0x03c6,), # 'ϕ': 'φ'
-    # GREEK PI SYMBOL: GREEK SMALL LETTER PI
-    0x03d6: (0x03c0,), # 'ϖ': 'π'
-    # GREEK KAPPA SYMBOL: GREEK SMALL LETTER KAPPA
-    0x03f0: (0x03ba,), # 'ϰ': 'κ'
-    # GREEK RHO SYMBOL: GREEK SMALL LETTER RHO
-    0x03f1: (0x03c1,), # 'ϱ': 'ρ'
-    # GREEK LUNATE EPSILON SYMBOL: GREEK SMALL LETTER EPSILON
-    0x03f5: (0x03b5,), # 'ϵ': 'ε'
-    # CYRILLIC SMALL LETTER VE: CYRILLIC SMALL LETTER ROUNDED VE
-    0x0432: (0x1c80,), # 'в': 'ᲀ'
-    # CYRILLIC SMALL LETTER DE: CYRILLIC SMALL LETTER LONG-LEGGED DE
-    0x0434: (0x1c81,), # 'д': 'ᲁ'
-    # CYRILLIC SMALL LETTER O: CYRILLIC SMALL LETTER NARROW O
-    0x043e: (0x1c82,), # 'о': 'ᲂ'
-    # CYRILLIC SMALL LETTER ES: CYRILLIC SMALL LETTER WIDE ES
-    0x0441: (0x1c83,), # 'с': 'ᲃ'
-    # CYRILLIC SMALL LETTER TE: CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
-    0x0442: (0x1c84, 0x1c85), # 'т': 'ᲄᲅ'
-    # CYRILLIC SMALL LETTER HARD SIGN: CYRILLIC SMALL LETTER TALL HARD SIGN
-    0x044a: (0x1c86,), # 'ъ': 'ᲆ'
-    # CYRILLIC SMALL LETTER YAT: CYRILLIC SMALL LETTER TALL YAT
-    0x0463: (0x1c87,), # 'ѣ': 'ᲇ'
-    # CYRILLIC SMALL LETTER ROUNDED VE: CYRILLIC SMALL LETTER VE
-    0x1c80: (0x0432,), # 'ᲀ': 'в'
-    # CYRILLIC SMALL LETTER LONG-LEGGED DE: CYRILLIC SMALL LETTER DE
-    0x1c81: (0x0434,), # 'ᲁ': 'д'
-    # CYRILLIC SMALL LETTER NARROW O: CYRILLIC SMALL LETTER O
-    0x1c82: (0x043e,), # 'ᲂ': 'о'
-    # CYRILLIC SMALL LETTER WIDE ES: CYRILLIC SMALL LETTER ES
-    0x1c83: (0x0441,), # 'ᲃ': 'с'
-    # CYRILLIC SMALL LETTER TALL TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
-    0x1c84: (0x0442, 0x1c85), # 'ᲄ': 'тᲅ'
-    # CYRILLIC SMALL LETTER THREE-LEGGED TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE
-    0x1c85: (0x0442, 0x1c84), # 'ᲅ': 'тᲄ'
-    # CYRILLIC SMALL LETTER TALL HARD SIGN: CYRILLIC SMALL LETTER HARD SIGN
-    0x1c86: (0x044a,), # 'ᲆ': 'ъ'
-    # CYRILLIC SMALL LETTER TALL YAT: CYRILLIC SMALL LETTER YAT
-    0x1c87: (0x0463,), # 'ᲇ': 'ѣ'
-    # CYRILLIC SMALL LETTER UNBLENDED UK: CYRILLIC SMALL LETTER MONOGRAPH UK
-    0x1c88: (0xa64b,), # 'ᲈ': 'ꙋ'
-    # LATIN SMALL LETTER S WITH DOT ABOVE: LATIN SMALL LETTER LONG S WITH DOT ABOVE
-    0x1e61: (0x1e9b,), # 'ṡ': 'ẛ'
-    # LATIN SMALL LETTER LONG S WITH DOT ABOVE: LATIN SMALL LETTER S WITH DOT ABOVE
-    0x1e9b: (0x1e61,), # 'ẛ': 'ṡ'
-    # GREEK PROSGEGRAMMENI: COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA
-    0x1fbe: (0x0345, 0x03b9), # 'ι': '\u0345ι'
-    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
-    0x1fd3: (0x0390,), # 'ΐ': 'ΐ'
-    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
-    0x1fe3: (0x03b0,), # 'ΰ': 'ΰ'
-    # CYRILLIC SMALL LETTER MONOGRAPH UK: CYRILLIC SMALL LETTER UNBLENDED UK
-    0xa64b: (0x1c88,), # 'ꙋ': 'ᲈ'
-    # LATIN SMALL LIGATURE LONG S T: LATIN SMALL LIGATURE ST
-    0xfb05: (0xfb06,), # 'ſt': 'st'
-    # LATIN SMALL LIGATURE ST: LATIN SMALL LIGATURE LONG S T
-    0xfb06: (0xfb05,), # 'st': 'ſt'
-}
+#
+# Every character which has to be matched case-insensitively with another one
+# is folded to the same code by sre_lower_unicode(), so this module has no
+# tables; the script checks that this holds.

Lib/re/_compiler.py

@@ -13,7 +13,6 @@
 import _sre
 from . import _parser
 from ._constants import *
-from ._casefix import _EXTRA_CASES
 from ._optimizer import (
     _combine_flags, _compile_charset, _optimize_charset, _compile_info,
     _simple, _CHARSET_ALL, _CODEBITS, MAXCODE, optimize,
@@ -41,12 +40,12 @@ def _compile(code, pattern, flags):
     ASSERT_CODES = _ASSERT_CODES
     iscased = None
     tolower = None
-    fixes = None
+    isunicode = False
     if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE:
         if flags & SRE_FLAG_UNICODE:
             iscased = _sre.unicode_iscased
             tolower = _sre.unicode_tolower
-            fixes = _EXTRA_CASES
+            isunicode = True
         else:
             iscased = _sre.ascii_iscased
             tolower = _sre.ascii_tolower
@@ -62,25 +61,10 @@ def _compile(code, pattern, flags):
                 emit(op)
                 emit(av)
             else:
-                lo = tolower(av)
-                if not fixes:  # ascii
-                    emit(OP_IGNORE[op])
-                    emit(lo)
-                elif lo not in fixes:
-                    emit(OP_UNICODE_IGNORE[op])
-                    emit(lo)
-                else:
-                    emit(IN_UNI_IGNORE)
-                    skip = _len(code); emit(0)
-                    if op is NOT_LITERAL:
-                        emit(NEGATE)
-                    for k in (lo,) + fixes[lo]:
-                        emit(LITERAL)
-                        emit(k)
-                    emit(FAILURE)
-                    code[skip] = _len(code) - skip
+                emit(OP_UNICODE_IGNORE[op] if isunicode else OP_IGNORE[op])
+                emit(tolower(av))
         elif op is IN:
-            charset, hascased = _optimize_charset(av, iscased, tolower, fixes)
+            charset, hascased = _optimize_charset(av, iscased, tolower, isunicode)
             if not charset:
                 emit(FAILURE)
             elif charset == _CHARSET_ALL:
@@ -90,7 +74,7 @@ def _compile(code, pattern, flags):
                     emit(IN_LOC_IGNORE)
                 elif not hascased:
                     emit(IN)
-                elif not fixes:  # ascii
+                elif not isunicode:
                     emit(IN_IGNORE)
                 else:
                     emit(IN_UNI_IGNORE)
@@ -192,7 +176,7 @@ def _compile(code, pattern, flags):
                 emit(op)
             elif flags & SRE_FLAG_LOCALE:
                 emit(GROUPREF_LOC_IGNORE)
-            elif not fixes:  # ascii
+            elif not isunicode:
                 emit(GROUPREF_IGNORE)
             else:
                 emit(GROUPREF_UNI_IGNORE)

Lib/re/_optimizer.py

@@ -55,7 +55,7 @@ def _compile_charset(charset, flags, code):
             raise PatternError(f"internal: unsupported set operator {op!r}")
     emit(FAILURE)
 
-def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
+def _optimize_charset(charset, iscased=None, fixup=None, isunicode=False):
     # internal: optimize character set.
     #
     # The engine's charset() walk toggles polarity on every NEGATE (see
@@ -67,7 +67,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
     if not negates or negates == [0]:
         # Fast path: a plain union, optionally complemented as a whole -- every
         # charset the parser produces today, optimized as before.
-        return _optimize_charset_segment(charset, iscased, fixup, fixes)
+        return _optimize_charset_segment(charset, iscased, fixup, isunicode)
 
     # Optimize each NEGATE-delimited run on its own.  _allow_anyall is off: the
     # [\s\S] -> ANY_ALL / [^\s\S] -> empty shortcuts rewrite a whole set and
@@ -78,16 +78,17 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
     for i in negates + [len(charset)]:
         if i > start:                  # skip an empty run (e.g. a leading NEGATE)
             opt, cased = _optimize_charset_segment(
-                charset[start:i], iscased, fixup, fixes, _allow_anyall=False)
+                charset[start:i], iscased, fixup, isunicode,
+                _allow_anyall=False)
             out.extend(opt)
             hascased |= cased
         if i < len(charset):
             out.append((NEGATE, None))
         start = i + 1
     return out, hascased
 
-def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None,
-                              _allow_anyall=True):
+def _optimize_charset_segment(charset, iscased=None, fixup=None,
+                              isunicode=False, _allow_anyall=True):
     # internal: optimize one NEGATE-free union of character-set members
     out = []
     tail = []
@@ -100,25 +101,15 @@ def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None,
                     if fixup: # IGNORECASE and not LOCALE
                         av = fixup(av)
                         charmap[av] = 1
-                        if fixes and av in fixes:
-                            for k in fixes[av]:
-                                charmap[k] = 1
                         if not hascased and iscased(av):
                             hascased = True
                     else:
                         charmap[av] = 1
                 elif op is RANGE:
                     r = range(av[0], av[1]+1)
                     if fixup: # IGNORECASE and not LOCALE
-                        if fixes:
-                            for i in map(fixup, r):
-                                charmap[i] = 1
-                                if i in fixes:
-                                    for k in fixes[i]:
-                                        charmap[k] = 1
-                        else:
-                            for i in map(fixup, r):
-                                charmap[i] = 1
+                        for i in map(fixup, r):
+                            charmap[i] = 1
                         if not hascased:
                             hascased = any(map(iscased, r))
                     else:
@@ -151,7 +142,7 @@ def _optimize_charset_segment(charset, iscased=None, fixup=None, fixes=None,
                     # Also, both c.lower() and c.lower().upper() are single
                     # characters for every non-BMP character.
                     if op is RANGE:
-                        if fixes: # not ASCII
+                        if isunicode:
                             op = RANGE_UNI_IGNORE
                         hascased = True
                     else:

Lib/test/test_re.py

@@ -1327,6 +1327,46 @@ def test_ignore_case(self):
         self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I))
         self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I))
 
+        # Cherokee letters case fold to their uppercase. Unicode 8.0+.
+        assert '\uab70'.casefold() == '\u13a0'.casefold() == '\u13a0' # 'ꭰ', 'Ꭰ'
+        self.assertTrue(re.match('\u13a0', '\uab70', re.I))
+        self.assertTrue(re.match('\uab70', '\u13a0', re.I))
+
+    # Characters which the matcher maps to the same code (see
+    # _sre.unicode_tolower), and which are therefore matched
+    # case-insensitively wherever it compares characters.
+    FOLDED_PAIRS = [
+        ('\u03c2', '\u03c3'),  # 'ς', 'σ'
+        ('\u03c3', '\u03a3'),  # 'σ', 'Σ'
+        ('\u00b5', '\u03bc'),  # 'µ', 'μ'
+        ('\u017f', 's'),      # 'ſ'
+        ('\u00df', '\u1e9e'),  # 'ß', 'ẞ'
+        ('k', '\u212a'),      # KELVIN SIGN
+        ('\u0432', '\u1c80'),  # 'в', 'ᲀ'
+        ('\uab70', '\u13a0'),  # 'ꭰ', 'Ꭰ'
+        # Not unified by the case folding (see sre_lower_unicode).
+        ('i', '\u0131'),      # LATIN SMALL LETTER DOTLESS I
+        ('\u0390', '\u1fd3'),  # 'ΐ' with tonos and with oxia
+        ('\u03b0', '\u1fe3'),  # 'ΰ' with tonos and with oxia
+        ('\ufb05', '\ufb06'),  # 'ſt', 'st'
+    ]
+
+    def test_ignore_case_backreference(self):
+        for a, b in self.FOLDED_PAIRS:
+            with self.subTest(pair=(a, b)):
+                self.assertTrue(re.fullmatch(r'(.)\1', a + b, re.I))
+                self.assertTrue(re.fullmatch(r'(.)\1', b + a, re.I))
+
+    def test_ignore_case_folded_pairs(self):
+        for a, b in self.FOLDED_PAIRS:
+            with self.subTest(pair=(a, b)):
+                self.assertTrue(re.match(a, b, re.I))
+                self.assertTrue(re.match(b, a, re.I))
+                self.assertTrue(re.match('[%s]' % a, b, re.I))
+                self.assertTrue(re.match('[%s]' % b, a, re.I))
+                self.assertIsNone(re.match('[^%s]' % a, b, re.I))
+                self.assertIsNone(re.match('[^%s]' % b, a, re.I))
+
     def test_ignore_case_set(self):
         self.assertTrue(re.match(r'[19A]', 'A', re.I))
         self.assertTrue(re.match(r'[19a]', 'a', re.I))
@@ -3196,11 +3236,18 @@ def test_case_helpers(self):
             self.assertEqual(_sre.ascii_iscased(i), iscased)
             self.assertEqual(_sre.unicode_iscased(i), iscased)
 
+        # Characters which the case folding does not unify.
+        fold_exceptions = {0x0131: 0x0069, 0x1fd3: 0x0390,
+                           0x1fe3: 0x03b0, 0xfb05: 0xfb06}
         for i in list(range(128, 0x1000)) + [0x10400, 0x10428]:
             c = chr(i)
             self.assertEqual(_sre.ascii_tolower(i), i)
-            if i != 0x0130:
-                self.assertEqual(_sre.unicode_tolower(i), ord(c.lower()))
+            if i != 0x0130 and i not in fold_exceptions:
+                # unicode_tolower() is the case folding when it is a single
+                # character, and the lowercase otherwise.
+                f = c.casefold()
+                self.assertEqual(_sre.unicode_tolower(i),
+                                 ord(f) if len(f) == 1 else ord(c.lower()))
             iscased = c != c.lower() or c != c.upper()
             self.assertFalse(_sre.ascii_iscased(i))
             self.assertEqual(_sre.unicode_iscased(i),
@@ -3211,6 +3258,22 @@ def test_case_helpers(self):
         self.assertFalse(_sre.ascii_iscased(0x0130))
         self.assertTrue(_sre.unicode_iscased(0x0130))
 
+        # Case folding, unlike lowercasing, unifies these.
+        self.assertEqual(_sre.unicode_tolower(0x00b5), 0x03bc)  # 'µ', 'μ'
+        self.assertEqual(_sre.unicode_tolower(0x017f), ord('s'))  # 'ſ'
+        # The full case folding of both sharp S characters is "ss", so they
+        # are matched through their common lowercase.
+        self.assertEqual(_sre.unicode_tolower(0x00df), 0x00df)  # 'ß'
+        self.assertEqual(_sre.unicode_tolower(0x1e9e), 0x00df)  # 'ẞ'
+        # Cherokee letters case fold to their uppercase, which is lowercased
+        # back so that the two forms share a key.
+        self.assertEqual(_sre.unicode_tolower(0x13a0), 0xab70)  # 'Ꭰ', 'ꭰ'
+        self.assertEqual(_sre.unicode_tolower(0xab70), 0xab70)
+        for i, lo in fold_exceptions.items():
+            with self.subTest(char=chr(i)):
+                self.assertEqual(_sre.unicode_tolower(i), lo)
+
+
     @cpython_only
     def test_dealloc(self):
         # issue 3299: check for segfault in debug build

Misc/NEWS.d/next/Library/2026-08-28-12-32-03.gh-issue-156513.hD62em.rst

@@ -0,0 +1,3 @@
+Fix case-insensitive matching of a backreference in :mod:`re`: it now matches
+the characters which are matched as a literal, so that ``(?i)(.)\1`` matches
+``'ςσ'``.

Modules/_sre/sre.c

@@ -214,9 +214,24 @@ static unsigned int sre_upper_locale(unsigned int ch)
 #define SRE_UNI_IS_PRINT(ch) ((SRE_UNI_IS_GRAPH(ch) || SRE_UNI_IS_BLANK(ch)) && \
                               !SRE_IS_CC(ch))
 
+/* The key by which characters are compared case-insensitively: the lowercase
+   of the simple case folding.  Folding, unlike lowercasing, makes MICRO SIGN
+   and GREEK SMALL LETTER MU compare equal; lowercasing the result keeps the
+   key below the case pairs, as Cherokee letters fold to their uppercase.
+   When the full folding is not a single character (as "ss" for both SHARP S
+   characters), the character is lowercased on its own. */
 static unsigned int sre_lower_unicode(unsigned int ch)
 {
-    return (unsigned int) Py_UNICODE_TOLOWER(ch);
+    switch (ch) {
+    /* Characters which the case folding does not unify: they only have the
+       same uppercase, or the same full folding of more than one character.
+       All are Unicode 1.1. */
+    case 0x0131: return 0x0069;  /* DOTLESS I -> I */
+    case 0x1fd3: return 0x0390;  /* IOTA WITH DIALYTIKA AND OXIA -> TONOS */
+    case 0x1fe3: return 0x03b0;  /* UPSILON WITH DIALYTIKA AND OXIA -> TONOS */
+    case 0xfb05: return 0xfb06;  /* LIGATURE LONG S T -> LIGATURE ST */
+    }
+    return (unsigned int) _PyUnicode_ToFoldedLowercase((Py_UCS4)ch);
 }
 
 static unsigned int sre_upper_unicode(unsigned int ch)

Modules/_sre/sre_lib.h

@@ -134,7 +134,7 @@ SRE(charset)(SRE_STATE* state, const SRE_CODE* set, SRE_CODE ch)
             /* <RANGE_UNI_IGNORE> <lower> <upper> */
         {
             SRE_CODE uch;
-            /* ch is already lower cased */
+            /* ch is already case folded (see sre_lower_unicode) */
             if (set[0] <= ch && ch <= set[1])
                 return ok;
             uch = sre_upper_unicode(ch);