diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst new file mode 100644 index 000000000000000..9e78e5ad545a83a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst @@ -0,0 +1,2 @@ +Speed up the parser by letting memoization lookups that cannot match return +immediately. diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f9950955..df83469895e03c0 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -90,6 +90,7 @@ _PyPegen_insert_memo(Parser *p, int mark, int type, void *node) m->mark = p->mark; m->next = p->tokens[mark]->memo; p->tokens[mark]->memo = m; + p->tokens[mark]->memo_mask |= 1ULL << (type & 63); return 0; } @@ -350,6 +351,10 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) Token *t = p->tokens[p->mark]; + if (!(t->memo_mask & (1ULL << (type & 63)))) { + return 0; + } + for (Memo *m = t->memo; m != NULL; m = m->next) { if (m->type == type) { #if defined(Py_DEBUG) diff --git a/Parser/pegen.h b/Parser/pegen.h index cbf44ba474fa395..12c038fa18a9080 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -42,6 +42,10 @@ typedef struct { int level; int lineno, col_offset, end_lineno, end_col_offset; Memo *memo; + // Filter over the rule types present in `memo` (bit `type & 63` is set + // for every entry): lets lookups skip walking the list on definite + // misses, which are the common case. + uint64_t memo_mask; PyObject *metadata; } Token;