gh-153569: move f-string state to source spans

Lib/test/test_capi/test_tokenizer.py

@@ -9,9 +9,6 @@ class TokenizerTests(unittest.TestCase):
     def test_source(self):
         _testinternalcapi.test_tokenizer_source()
 
-    def test_cursor(self):
-        _testinternalcapi.test_tokenizer_cursor()
-
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_fstring.py

@@ -1679,6 +1679,16 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+        d = {'a#b': 42}
+        self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+                         'f"{d["a#b"]}"=\'42\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')

Lib/test/test_tokenize.py

@@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_buffer_reallocation(self):
+        padding = " " * 9000
+        expression_line = ")=:>{2}}\n"
+        physical_lines = [
+            'f"""\n',
+            "{(\n",
+            padding + "1\n",
+            expression_line,
+            '"""\n',
+        ]
+        source = "".join(physical_lines)
+        chunks = iter([
+            "".join(physical_lines[:2]),
+            "".join(physical_lines[2:4]),
+            physical_lines[4],
+            "",
+        ])
+
+        expected = self._get_tokens(source, extra_tokens=True)
+        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+            chunks.__next__,
+            extra_tokens=True,
+        ))
+        self.assertEqual(tokens, expected)
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (

Lib/test/test_tstring.py

@@ -287,5 +287,17 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+        d = {'a#b': 42}
+        t = t'''{f"{d["a#b"]}"}'''
+        self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
 if __name__ == '__main__':
     unittest.main()

Makefile.pre.in

@@ -399,7 +399,6 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
-		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -415,7 +414,6 @@ TOKENIZER_HEADERS= \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
-		Parser/tokenizer/cursor.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3462,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h