From ba74b510af4166766f2cea8e16de8a115cf67a65 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 20 Jul 2026 17:39:00 +0200 Subject: [PATCH 1/3] fix endless loop --- crates/sparse-ngrams/Cargo.toml | 4 ++-- crates/sparse-ngrams/src/query.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sparse-ngrams/Cargo.toml b/crates/sparse-ngrams/Cargo.toml index 2da4bc9..d1214e2 100644 --- a/crates/sparse-ngrams/Cargo.toml +++ b/crates/sparse-ngrams/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" description = "Fast sparse n-gram extraction from byte slices." repository = "https://github.com/github/rust-gems" license = "MIT" -keywords = ["sparse-ngram", "ngram", "algorithm", "search", "code-search", "regular-expression"] -categories = ["algorithms", "data-structures", "text-processing", "search-engine", "data-retrieval"] +keywords = ["sparse-ngram", "ngram", "algorithm", "search", "regular-expression"] +categories = ["algorithms", "data-structures", "text-processing"] # `data/` holds the reference bigram ranking used only for offline model training/verification; # it is not needed to build or use the crate, so keep it out of the published package. exclude = ["data/"] diff --git a/crates/sparse-ngrams/src/query.rs b/crates/sparse-ngrams/src/query.rs index 2826c2c..6d4b8b9 100644 --- a/crates/sparse-ngrams/src/query.rs +++ b/crates/sparse-ngrams/src/query.rs @@ -304,9 +304,9 @@ impl QueryGrams { self.queue.clear(); if self.content_end_idx > 1 { // `self.h` already holds `bigram_h(last)` by invariant, so it needs no update. - let last = (self.content & 0xFF) as u8; - self.content = last as u64; self.content_end_idx = 1; + } else { + self.content_end_idx = 0; } } } From f002528b66ec952b23c7be38a830665322e4f2e1 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 20 Jul 2026 17:47:30 +0200 Subject: [PATCH 2/3] Update query.rs --- crates/sparse-ngrams/src/query.rs | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/sparse-ngrams/src/query.rs b/crates/sparse-ngrams/src/query.rs index 6d4b8b9..51052fc 100644 --- a/crates/sparse-ngrams/src/query.rs +++ b/crates/sparse-ngrams/src/query.rs @@ -304,8 +304,16 @@ impl QueryGrams { self.queue.clear(); if self.content_end_idx > 1 { // `self.h` already holds `bigram_h(last)` by invariant, so it needs no update. + // Keep just the single trailing byte for now: it is the left half of the next + // bigram, so a `consume_first` that stops here still satisfies + // `retained-byte + suffix == fresh(retained-byte + suffix)`. self.content_end_idx = 1; } else { + // A further `consume_first` on that lone byte drops it, converging to the default + // state. This fixpoint is required by the regex state-set reducer, which shrinks + // distinct states until they collapse into a shared one (see + // `consume_first_converges_to_default`). `state()` masks `content`, so the stale + // high bytes left in `self.content` are invisible and need not be cleared. self.content_end_idx = 0; } } @@ -799,4 +807,29 @@ mod tests { } } } + + /// Repeatedly calling `consume_first` (without any intervening `append`) must eventually reach + /// the default state. Consumers such as the regex state-set reducer shrink a *set* of states by + /// calling `consume_first` on the lowest-priority ones until distinct states collapse into a + /// shared one; if a state could get stuck at a non-default fixed point, two such states with + /// different retained bytes would never merge and the reducer would loop forever. + #[test] + fn consume_first_converges_to_default() { + for input in ["abc", "abcdef", "hello world", "ababababab", "mississippi"] { + let mut q = QueryGrams::default(); + for c in input.chars() { + q.append_char(c, |_gram, _idx, _follow| {}); + } + // Far more calls than there are bytes: the state must be at the default fixpoint well + // before this loop ends, and further calls must keep it there. + for _ in 0..(input.len() + 8) { + q.consume_first(|_gram, _idx, _follow| {}); + } + assert_eq!( + q.state(), + QueryGrams::default().state(), + "consume_first did not converge to the default state for input {input:?}", + ); + } + } } From 01295a3716bb010ca6c15e256f89dce867f88b92 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 20 Jul 2026 19:05:35 +0200 Subject: [PATCH 3/3] Update query.rs --- crates/sparse-ngrams/src/query.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/sparse-ngrams/src/query.rs b/crates/sparse-ngrams/src/query.rs index 51052fc..f7d3a64 100644 --- a/crates/sparse-ngrams/src/query.rs +++ b/crates/sparse-ngrams/src/query.rs @@ -166,10 +166,17 @@ impl QueryGrams { (content_len, self.content & mask) } - /// Returns the smallest active boundary priority. + /// Returns the smallest active boundary priority, or `u32::MAX` when there is no active + /// boundary left to consume. + /// + /// Callers use this to repeatedly drain the lowest-priority boundary across a *set* of states + /// (e.g. the regex state-set reducer). A state with an empty queue has nothing left to consume, + /// so it must report the *maximum* priority: otherwise it would be selected ahead of states that + /// can still make progress, `consume_first` on it would be a no-op, and the reducer would loop + /// forever. pub fn min_priority(&self) -> u32 { if self.queue.is_empty() { - 0 + u32::MAX } else { self.queue.front_value() }