From ac8954d12ebafaf0419e86fac6bce3850c2dd817 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 12:04:25 -0700 Subject: [PATCH 1/7] #171 Phase A: pure-JS substrate query engine + parity-pinned FNV-1a MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assets/js/search_substrate.js — the browser query pipeline as PURE functions (no DuckDB, no fetch, no DOM), so every piece of query logic is Node-testable and the explorer's ?fts=v1 wiring (Phase B) stays thin: - fnv1a32: 32-bit FNV-1a with a shift-decomposed multiply (no BigInt); parity with tools/build_search_index.py pinned by tests/search_fnv1a_regression.json (values generated from Python, fixtures include CJK/Cyrillic/Greek inputs). - planQuery: tokenize → query-time stopword drop (contract §3 list) → duplicate-term dedup → shard/hot-file resolution (hot_tokens.json manifest aware) with cross-token file dedup. Distinguishes 'all stopwords' (controlled empty state) from 'no tokens'. - bm25Contribution: k1=1.2 b=0.75, contract §5 field weights. - combineAndRank: AND semantics, per-pid score summation, stable ordering (score desc, pid asc), TOP_K=50. 12/12 unit tests, incl. the #172 hard-fail preconditions: stopword near-equivalence ('pottery from Cyprus' plans identically to 'pottery Cyprus'), duplicate-term identity, all-stopword controlled empty, hot-vs-base file resolution. Phase B (explorer.qmd ?fts=v1 flag wiring + benchmark harness) follows separately. Stacked on #329 (feat/170-search-index-builder). Refs #171, #170, #169. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- assets/js/search_substrate.js | 132 +++++++++++++++++++++++++++ tests/search_fnv1a_regression.json | 82 +++++++++++++++++ tests/unit/search-substrate.test.mjs | 127 ++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 assets/js/search_substrate.js create mode 100644 tests/search_fnv1a_regression.json create mode 100644 tests/unit/search-substrate.test.mjs diff --git a/assets/js/search_substrate.js b/assets/js/search_substrate.js new file mode 100644 index 0000000..609cb2d --- /dev/null +++ b/assets/js/search_substrate.js @@ -0,0 +1,132 @@ +// Browser-side query engine for the v1 search substrate (#171, SEARCH_INDEX_V1.md). +// +// PURE functions only — no DuckDB, no fetch, no DOM. The explorer's flag +// path (Phase B) feeds this module rows it fetched via db.query; that split +// keeps every piece of query logic unit-testable in Node +// (tests/unit/search-substrate.test.mjs) and the explorer wiring thin. +// +// Pipeline (contract §3, §5): +// tokenize (search_tokenizer.js) → drop query-time stopwords → locate each +// token's shard file(s) → BM25 per (pid, field) posting → field-weighted +// sum per pid → AND across tokens → top-K. + +import { tokenize } from './search_tokenizer.js'; + +// FNV-1a 32-bit over UTF-8 bytes. MUST match tools/build_search_index.py's +// fnv1a32 exactly — parity is pinned by tests/search_fnv1a_regression.json +// (values generated from the Python implementation). +export function fnv1a32(str) { + let h = 0x811c9dc5; + const bytes = new TextEncoder().encode(str); + for (const b of bytes) { + h ^= b; + // 32-bit multiply by the FNV prime 0x01000193 without BigInt: + // h * 16777619 = h*2^24 + h*2^9 + h*2^8 + h*2^7 + h*2^4 + h*2 + h + h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0; + } + return h >>> 0; +} + +// Curated query-time stopword list (contract §3). Build-time indexes +// EVERYTHING; dropping happens here only, so the policy stays reversible. +export const QUERY_STOPWORDS = new Set([ + 'a', 'an', 'the', 'of', 'from', 'for', 'to', 'in', 'on', 'at', + 'is', 'was', 'with', 'and', 'or', +]); + +// Field weights (contract §5) + BM25 constants. +export const FIELD_WEIGHTS = { + 'sample.label': 3.0, + 'concept.label': 2.5, + 'sample.place_name': 2.0, + 'sample.description': 1.0, +}; +export const BM25_K1 = 1.2; +export const BM25_B = 0.75; +export const TOP_K = 50; + +/** + * Plan a query: tokenize, apply stopword policy, dedupe, and resolve the + * substrate file path(s) for each surviving token. + * + * @param term raw user input + * @param manifest { shardCount, hotTokens } — shardCount from + * build_stats.json; hotTokens = the `tokens` object of + * hot_tokens.json ({token: {key, sub_files}}). + * @returns {{ + * tokens: string[], // surviving, deduped, order-preserved + * allStopwords: boolean, // input had tokens but none survived (§3 + * // controlled-empty state, NOT a search) + * empty: boolean, // input produced no tokens at all + * files: string[], // deduped relative substrate paths to fetch + * filesByToken: Map, + * }} + */ +export function planQuery(term, manifest) { + const raw = tokenize(term); + const survivors = []; + for (const t of raw) { + if (QUERY_STOPWORDS.has(t)) continue; + if (!survivors.includes(t)) survivors.push(t); // duplicate-term dedup + } + const filesByToken = new Map(); + const files = []; + for (const t of survivors) { + const hot = manifest.hotTokens && manifest.hotTokens[t]; + const tokenFiles = hot + ? Array.from({ length: hot.sub_files }, + (_, m) => `hot/${hot.key}_p${m}.parquet`) + : [`shard_${String(fnv1a32(t) % manifest.shardCount).padStart(3, '0')}.parquet`]; + filesByToken.set(t, tokenFiles); + for (const f of tokenFiles) if (!files.includes(f)) files.push(f); + } + return { + tokens: survivors, + allStopwords: raw.length > 0 && survivors.length === 0, + empty: raw.length === 0, + files, + filesByToken, + }; +} + +/** + * BM25 contribution of one posting row (contract §5). + * @param row { field, tf, doc_len, df } + * @param stats { totalDocs, avgDocLen } — corpus-level; #171 Phase B reads + * totalDocs as the df.parquet row-domain (COUNT of (pid,field) + * docs) and avgDocLen over the fetched postings' universe. + */ +export function bm25Contribution(row, stats) { + const idf = Math.log((stats.totalDocs - row.df + 0.5) / (row.df + 0.5) + 1); + const norm = row.tf * (BM25_K1 + 1) + / (row.tf + BM25_K1 * (1 - BM25_B + BM25_B * row.doc_len / stats.avgDocLen)); + const weight = FIELD_WEIGHTS[row.field] ?? 1.0; + return weight * idf * norm; +} + +/** + * Combine postings into the final AND-ranked top-K. + * + * @param postingsByToken Map> + * @param stats { totalDocs, avgDocLen } + * @param k top-K cap (default contract TOP_K = 50) + * @returns Array<{pid, score}> sorted score desc, pid asc for stable ties. + */ +export function combineAndRank(postingsByToken, stats, k = TOP_K) { + const scores = new Map(); // pid -> summed score + const matched = new Map(); // pid -> Set + for (const [token, rows] of postingsByToken) { + for (const row of rows) { + scores.set(row.pid, (scores.get(row.pid) ?? 0) + bm25Contribution(row, stats)); + if (!matched.has(row.pid)) matched.set(row.pid, new Set()); + matched.get(row.pid).add(token); + } + } + const need = postingsByToken.size; // AND semantics: every token matches + const out = []; + for (const [pid, tokens] of matched) { + if (tokens.size === need) out.push({ pid, score: scores.get(pid) }); + } + out.sort((a, b) => (b.score - a.score) || (a.pid < b.pid ? -1 : 1)); + return out.slice(0, k); +} diff --git a/tests/search_fnv1a_regression.json b/tests/search_fnv1a_regression.json new file mode 100644 index 0000000..d818159 --- /dev/null +++ b/tests/search_fnv1a_regression.json @@ -0,0 +1,82 @@ +[ + { + "input": "pottery", + "fnv1a32": 3121265474 + }, + { + "input": "cyprus", + "fnv1a32": 1194896787 + }, + { + "input": "basalt", + "fnv1a32": 3758865220 + }, + { + "input": "catalhoyuk", + "fnv1a32": 1680059526 + }, + { + "input": "material", + "fnv1a32": 3538210912 + }, + { + "input": "a", + "fnv1a32": 3826002220 + }, + { + "input": "z", + "fnv1a32": 4278997933 + }, + { + "input": "igsn", + "fnv1a32": 877601424 + }, + { + "input": "hrv000abc", + "fnv1a32": 4165183063 + }, + { + "input": "1965", + "fnv1a32": 676065338 + }, + { + "input": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "fnv1a32": 3516603077 + }, + { + "input": "陶器", + "fnv1a32": 1823698683 + }, + { + "input": "сосуд", + "fnv1a32": 3730779371 + }, + { + "input": "κεραμικος", + "fnv1a32": 2429063684 + }, + { + "input": "unknownstuff", + "fnv1a32": 1666634825 + }, + { + "input": "volcanic", + "fnv1a32": 1893227256 + }, + { + "input": "rock", + "fnv1a32": 974867124 + }, + { + "input": "seamount", + "fnv1a32": 2199515177 + }, + { + "input": "0", + "fnv1a32": 890022063 + }, + { + "input": "9999999", + "fnv1a32": 202886326 + } +] \ No newline at end of file diff --git a/tests/unit/search-substrate.test.mjs b/tests/unit/search-substrate.test.mjs new file mode 100644 index 0000000..d3508b2 --- /dev/null +++ b/tests/unit/search-substrate.test.mjs @@ -0,0 +1,127 @@ +// Unit tests for assets/js/search_substrate.js (#171 Phase A). +// FNV-1a parity values come from tools/build_search_index.py (fixture +// generated by the Python implementation — same pattern as the tokenizer +// regression set). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +import { + fnv1a32, planQuery, bm25Contribution, combineAndRank, + QUERY_STOPWORDS, FIELD_WEIGHTS, BM25_K1, BM25_B, +} from '../../assets/js/search_substrate.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const FNV = JSON.parse(readFileSync(join(here, '..', 'search_fnv1a_regression.json'), 'utf8')); + +test('fnv1a32 parity with Python (20-entry fixture incl. CJK/Cyrillic/Greek)', () => { + for (const { input, fnv1a32: expected } of FNV) { + assert.equal(fnv1a32(input), expected, `fnv1a32(${JSON.stringify(input)})`); + } +}); + +const MANIFEST = { + shardCount: 256, + hotTokens: { material: { key: 'deadbeef', sub_files: 3 } }, +}; + +test('planQuery: stopword policy + duplicate-term dedup', () => { + const p1 = planQuery('pottery from Cyprus', MANIFEST); + assert.deepEqual(p1.tokens, ['pottery', 'cyprus']); // 'from' dropped + const p2 = planQuery('pottery pottery cyprus', MANIFEST); + assert.deepEqual(p2.tokens, ['pottery', 'cyprus']); // dupes collapse + // #172 hard-fail precondition: both plans fetch identical files. + assert.deepEqual(p1.files, p2.files); +}); + +test('planQuery: all-stopword input → controlled empty, no fetches', () => { + const p = planQuery('a the of', MANIFEST); + assert.equal(p.allStopwords, true); + assert.equal(p.empty, false); + assert.deepEqual(p.files, []); +}); + +test('planQuery: empty/whitespace input', () => { + const p = planQuery(' ', MANIFEST); + assert.equal(p.empty, true); + assert.equal(p.allStopwords, false); +}); + +test('planQuery: base-shard path uses fnv1a32 % shardCount, zero-padded', () => { + const p = planQuery('pottery', MANIFEST); + const shard = fnv1a32('pottery') % 256; + assert.deepEqual(p.files, [`shard_${String(shard).padStart(3, '0')}.parquet`]); +}); + +test('planQuery: hot token resolves to its sub-file set', () => { + const p = planQuery('material', MANIFEST); + assert.deepEqual(p.files, [ + 'hot/deadbeef_p0.parquet', 'hot/deadbeef_p1.parquet', 'hot/deadbeef_p2.parquet', + ]); +}); + +test('planQuery: shared shard between tokens is fetched once', () => { + // Construct two tokens that collide mod 2. + const m = { shardCount: 1, hotTokens: {} }; + const p = planQuery('pottery basalt', m); + assert.deepEqual(p.files, ['shard_000.parquet']); + assert.equal(p.filesByToken.get('pottery')[0], 'shard_000.parquet'); +}); + +test('bm25Contribution: hand-computed value', () => { + // df=10 of 1000 docs, tf=2, doc_len=4, avg=8, field weight 3.0 + const row = { field: 'sample.label', tf: 2, doc_len: 4, df: 10 }; + const stats = { totalDocs: 1000, avgDocLen: 8 }; + const idf = Math.log((1000 - 10 + 0.5) / (10 + 0.5) + 1); + const norm = (2 * 2.2) / (2 + 1.2 * (1 - 0.75 + 0.75 * 4 / 8)); + assert.ok(Math.abs(bm25Contribution(row, stats) - 3.0 * idf * norm) < 1e-12); +}); + +test('bm25Contribution: rarer token scores higher, heavier field scores higher', () => { + const stats = { totalDocs: 1000, avgDocLen: 8 }; + const base = { field: 'sample.description', tf: 1, doc_len: 8 }; + const rare = bm25Contribution({ ...base, df: 2 }, stats); + const common = bm25Contribution({ ...base, df: 500 }, stats); + assert.ok(rare > common); + const label = bm25Contribution({ ...base, df: 10, field: 'sample.label' }, stats); + const desc = bm25Contribution({ ...base, df: 10 }, stats); + assert.ok(Math.abs(label / desc - 3.0) < 1e-9); // weight ratio 3.0 : 1.0 +}); + +test('combineAndRank: AND semantics + score summation + stable ties', () => { + const stats = { totalDocs: 1000, avgDocLen: 8 }; + const row = (pid, field, df) => ({ pid, field, tf: 1, doc_len: 8, df }); + const postings = new Map([ + ['pottery', [row('p1', 'sample.label', 10), row('p2', 'sample.label', 10), + row('p3', 'concept.label', 10)]], + ['cyprus', [row('p1', 'sample.description', 20), row('p3', 'sample.place_name', 20)]], + ]); + const out = combineAndRank(postings, stats); + // p2 matched only 'pottery' → excluded (AND). + assert.deepEqual(out.map(r => r.pid).sort(), ['p1', 'p3']); + // p1: label(3.0) + description(1.0); p3: concept(2.5) + place(2.0) → p3 wins. + assert.equal(out[0].pid, 'p3'); + // Ties break by pid ascending. + const tied = combineAndRank(new Map([ + ['x', [row('b', 'sample.label', 10), row('a', 'sample.label', 10)]], + ]), stats); + assert.deepEqual(tied.map(r => r.pid), ['a', 'b']); +}); + +test('combineAndRank: top-K cap', () => { + const stats = { totalDocs: 1000, avgDocLen: 8 }; + const rows = Array.from({ length: 80 }, (_, i) => + ({ pid: `p${String(i).padStart(2, '0')}`, field: 'sample.label', + tf: 1, doc_len: 8, df: 10 })); + const out = combineAndRank(new Map([['t', rows]]), stats); + assert.equal(out.length, 50); +}); + +test('constants match the contract', () => { + assert.equal(BM25_K1, 1.2); + assert.equal(BM25_B, 0.75); + assert.deepEqual(FIELD_WEIGHTS['sample.label'], 3.0); + assert.ok(QUERY_STOPWORDS.has('from') && !QUERY_STOPWORDS.has('pottery')); +}); From b1c955dc75e5d82b3799487a7d425e15826849f1 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 17:49:11 -0700 Subject: [PATCH 2/7] #171: rework substrate query engine to the final two-tier hot policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync with #329's merged contract (§6 two-tier rule + round-5 df embedding): - planQuery returns a mode ('empty'|'allStopwords'|'normal'|'topk'): non-fetchable common terms are dropped from the AND with an ignoredCommon disclosure list (UI must show it); fetchable hot tokens join the AND via their hot/ sub-files; all-common queries plan a single hot_topk.parquet fetch. - expectedBytes computed up-front from shard_sizes.json + manifest total_bytes (§7 transfer accounting; feeds the benchmark metric). - bm25Contribution takes {totalDocs, avgDocLenByField} from build_stats.json (per-field corpus averages — matches the builder's hot_topk scoring exactly); df arrives embedded in rows. 15 unit tests incl. the two-tier cases (drop-with-disclosure, fetchable joins AND, all-common topk mode, expectedBytes accounting). 110/110 across the repo. Refs #171. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- assets/js/search_substrate.js | 118 +++++++++++++++++++-------- tests/unit/search-substrate.test.mjs | 116 +++++++++++++++++--------- 2 files changed, 158 insertions(+), 76 deletions(-) diff --git a/assets/js/search_substrate.js b/assets/js/search_substrate.js index 609cb2d..b1bd7b6 100644 --- a/assets/js/search_substrate.js +++ b/assets/js/search_substrate.js @@ -1,19 +1,21 @@ // Browser-side query engine for the v1 search substrate (#171, SEARCH_INDEX_V1.md). // // PURE functions only — no DuckDB, no fetch, no DOM. The explorer's flag -// path (Phase B) feeds this module rows it fetched via db.query; that split -// keeps every piece of query logic unit-testable in Node +// path feeds this module data it fetched via db.query; the split keeps every +// piece of query logic unit-testable in Node // (tests/unit/search-substrate.test.mjs) and the explorer wiring thin. // -// Pipeline (contract §3, §5): -// tokenize (search_tokenizer.js) → drop query-time stopwords → locate each -// token's shard file(s) → BM25 per (pid, field) posting → field-weighted -// sum per pid → AND across tokens → top-K. +// Pipeline (contract §3, §5, §6 two-tier rule): +// tokenize (search_tokenizer.js) → drop query-time stopwords → dedupe → +// two-tier hot policy (fetchable hot joins the AND; non-fetchable common +// terms are dropped-with-disclosure, or an all-common query serves from +// the hot_topk sidecar) → resolve substrate files → BM25 per posting → +// field-weighted per-pid sums → AND across tokens → top-K. import { tokenize } from './search_tokenizer.js'; // FNV-1a 32-bit over UTF-8 bytes. MUST match tools/build_search_index.py's -// fnv1a32 exactly — parity is pinned by tests/search_fnv1a_regression.json +// fnv1a32 exactly — parity pinned by tests/search_fnv1a_regression.json // (values generated from the Python implementation). export function fnv1a32(str) { let h = 0x811c9dc5; @@ -21,7 +23,6 @@ export function fnv1a32(str) { for (const b of bytes) { h ^= b; // 32-bit multiply by the FNV prime 0x01000193 without BigInt: - // h * 16777619 = h*2^24 + h*2^9 + h*2^8 + h*2^7 + h*2^4 + h*2 + h h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0; } return h >>> 0; @@ -45,61 +46,106 @@ export const BM25_K1 = 1.2; export const BM25_B = 0.75; export const TOP_K = 50; +const shardFile = (token, shardCount) => + `shard_${String(fnv1a32(token) % shardCount).padStart(3, '0')}.parquet`; + /** - * Plan a query: tokenize, apply stopword policy, dedupe, and resolve the - * substrate file path(s) for each surviving token. + * Plan a query under the §6 two-tier hot rule. * - * @param term raw user input - * @param manifest { shardCount, hotTokens } — shardCount from - * build_stats.json; hotTokens = the `tokens` object of - * hot_tokens.json ({token: {key, sub_files}}). + * @param term raw user input + * @param manifest { shardCount, hotTokens } — shardCount from + * build_stats.json; hotTokens = hot_tokens.json's `tokens` + * ({token: {key, sub_files, postings, total_bytes, + * fetchable}}). + * @param shardSizes optional shard_sizes.json object ({file: bytes}); when + * given, the plan carries expectedBytes so callers can + * report transfer up-front (§7). * @returns {{ - * tokens: string[], // surviving, deduped, order-preserved - * allStopwords: boolean, // input had tokens but none survived (§3 - * // controlled-empty state, NOT a search) - * empty: boolean, // input produced no tokens at all + * mode: 'empty'|'allStopwords'|'normal'|'topk', + * tokens: string[], // tokens participating in the AND + * ignoredCommon: string[], // non-fetchable hot terms dropped (§3 — + * // the UI MUST disclose these) * files: string[], // deduped relative substrate paths to fetch * filesByToken: Map, + * expectedBytes: number|null, * }} */ -export function planQuery(term, manifest) { +export function planQuery(term, manifest, shardSizes = null) { const raw = tokenize(term); const survivors = []; for (const t of raw) { if (QUERY_STOPWORDS.has(t)) continue; if (!survivors.includes(t)) survivors.push(t); // duplicate-term dedup } + const base = { ignoredCommon: [], files: [], filesByToken: new Map(), expectedBytes: null }; + if (raw.length === 0) return { ...base, mode: 'empty', tokens: [] }; + if (survivors.length === 0) return { ...base, mode: 'allStopwords', tokens: [] }; + + const hot = manifest.hotTokens || {}; + const participating = []; + const common = []; + for (const t of survivors) { + const h = hot[t]; + if (h && !h.fetchable) common.push(t); + else participating.push(t); + } + + if (participating.length === 0) { + // Every surviving term is a common term: rank via the sidecar. + return { + mode: 'topk', tokens: common, ignoredCommon: [], + files: ['hot_topk.parquet'], + filesByToken: new Map(common.map(t => [t, ['hot_topk.parquet']])), + expectedBytes: null, + }; + } + const filesByToken = new Map(); const files = []; - for (const t of survivors) { - const hot = manifest.hotTokens && manifest.hotTokens[t]; - const tokenFiles = hot - ? Array.from({ length: hot.sub_files }, - (_, m) => `hot/${hot.key}_p${m}.parquet`) - : [`shard_${String(fnv1a32(t) % manifest.shardCount).padStart(3, '0')}.parquet`]; + for (const t of participating) { + const h = hot[t]; + const tokenFiles = (h && h.fetchable) + ? Array.from({ length: h.sub_files }, + (_, m) => `hot/${h.key}_p${m}.parquet`) + : [shardFile(t, manifest.shardCount)]; filesByToken.set(t, tokenFiles); for (const f of tokenFiles) if (!files.includes(f)) files.push(f); } + let expectedBytes = null; + if (shardSizes) { + expectedBytes = 0; + const counted = new Set(); + for (const t of participating) { + const h = hot[t]; + if (h && h.fetchable) { + if (!counted.has(h.key)) { expectedBytes += h.total_bytes; counted.add(h.key); } + } else { + const f = shardFile(t, manifest.shardCount); + if (!counted.has(f)) { expectedBytes += shardSizes[f] ?? 0; counted.add(f); } + } + } + } return { - tokens: survivors, - allStopwords: raw.length > 0 && survivors.length === 0, - empty: raw.length === 0, - files, - filesByToken, + mode: 'normal', tokens: participating, ignoredCommon: common, + files, filesByToken, expectedBytes, }; } /** * BM25 contribution of one posting row (contract §5). - * @param row { field, tf, doc_len, df } - * @param stats { totalDocs, avgDocLen } — corpus-level; #171 Phase B reads - * totalDocs as the df.parquet row-domain (COUNT of (pid,field) - * docs) and avgDocLen over the fetched postings' universe. + * @param row { field, tf, doc_len, df } — df is EMBEDDED in shipped rows + * (round-5 amendment; df.parquet is offline-only). + * @param stats { totalDocs, avgDocLenByField } — from build_stats.json: + * totalDocs = total_documents (distinct (pid, field) docs); + * avgDocLenByField[field] = fields[field].avg_doc_len + * (PER-FIELD corpus averages, matching the builder's + * hot_topk scoring). */ export function bm25Contribution(row, stats) { const idf = Math.log((stats.totalDocs - row.df + 0.5) / (row.df + 0.5) + 1); + const avgDl = stats.avgDocLenByField[row.field]; const norm = row.tf * (BM25_K1 + 1) - / (row.tf + BM25_K1 * (1 - BM25_B + BM25_B * row.doc_len / stats.avgDocLen)); + / (row.tf + BM25_K1 * (1 - BM25_B + BM25_B * row.doc_len / avgDl)); const weight = FIELD_WEIGHTS[row.field] ?? 1.0; return weight * idf * norm; } @@ -108,7 +154,7 @@ export function bm25Contribution(row, stats) { * Combine postings into the final AND-ranked top-K. * * @param postingsByToken Map> - * @param stats { totalDocs, avgDocLen } + * @param stats { totalDocs, avgDocLenByField } * @param k top-K cap (default contract TOP_K = 50) * @returns Array<{pid, score}> sorted score desc, pid asc for stable ties. */ diff --git a/tests/unit/search-substrate.test.mjs b/tests/unit/search-substrate.test.mjs index d3508b2..9cfa52d 100644 --- a/tests/unit/search-substrate.test.mjs +++ b/tests/unit/search-substrate.test.mjs @@ -1,7 +1,8 @@ -// Unit tests for assets/js/search_substrate.js (#171 Phase A). +// Unit tests for assets/js/search_substrate.js (#171). // FNV-1a parity values come from tools/build_search_index.py (fixture // generated by the Python implementation — same pattern as the tokenizer -// regression set). +// regression set). The manifest shapes mirror hot_tokens.json's two-tier +// schema (fetchable vs common terms, contract §6). import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; @@ -16,19 +17,27 @@ import { const here = dirname(fileURLToPath(import.meta.url)); const FNV = JSON.parse(readFileSync(join(here, '..', 'search_fnv1a_regression.json'), 'utf8')); -test('fnv1a32 parity with Python (20-entry fixture incl. CJK/Cyrillic/Greek)', () => { +test('fnv1a32 parity with Python (fixture incl. CJK/Cyrillic/Greek)', () => { for (const { input, fnv1a32: expected } of FNV) { assert.equal(fnv1a32(input), expected, `fnv1a32(${JSON.stringify(input)})`); } }); +// Two-tier manifest: 'material' is a true common term (non-fetchable); +// 'island' is storage-hot but fetchable (joins the AND normally). const MANIFEST = { shardCount: 256, - hotTokens: { material: { key: 'deadbeef', sub_files: 3 } }, + hotTokens: { + material: { key: 'deadbeef', sub_files: 10, postings: 5_000_000, + total_bytes: 50_000_000, fetchable: false }, + island: { key: 'cafe0001', sub_files: 2, postings: 82_000, + total_bytes: 2_500_000, fetchable: true }, + }, }; test('planQuery: stopword policy + duplicate-term dedup', () => { const p1 = planQuery('pottery from Cyprus', MANIFEST); + assert.equal(p1.mode, 'normal'); assert.deepEqual(p1.tokens, ['pottery', 'cyprus']); // 'from' dropped const p2 = planQuery('pottery pottery cyprus', MANIFEST); assert.deepEqual(p2.tokens, ['pottery', 'cyprus']); // dupes collapse @@ -38,15 +47,12 @@ test('planQuery: stopword policy + duplicate-term dedup', () => { test('planQuery: all-stopword input → controlled empty, no fetches', () => { const p = planQuery('a the of', MANIFEST); - assert.equal(p.allStopwords, true); - assert.equal(p.empty, false); + assert.equal(p.mode, 'allStopwords'); assert.deepEqual(p.files, []); }); test('planQuery: empty/whitespace input', () => { - const p = planQuery(' ', MANIFEST); - assert.equal(p.empty, true); - assert.equal(p.allStopwords, false); + assert.equal(planQuery(' ', MANIFEST).mode, 'empty'); }); test('planQuery: base-shard path uses fnv1a32 % shardCount, zero-padded', () => { @@ -55,67 +61,97 @@ test('planQuery: base-shard path uses fnv1a32 % shardCount, zero-padded', () => assert.deepEqual(p.files, [`shard_${String(shard).padStart(3, '0')}.parquet`]); }); -test('planQuery: hot token resolves to its sub-file set', () => { +test('planQuery two-tier: common term dropped WITH disclosure when selective terms exist', () => { + const p = planQuery('material basalt', MANIFEST); + assert.equal(p.mode, 'normal'); + assert.deepEqual(p.tokens, ['basalt']); // AND excludes 'material' + assert.deepEqual(p.ignoredCommon, ['material']); // UI must disclose this + assert.ok(!p.files.some(f => f.startsWith('hot/'))); // no hot bytes fetched +}); + +test('planQuery two-tier: fetchable hot joins the AND via its sub-files', () => { + const p = planQuery('island basalt', MANIFEST); + assert.equal(p.mode, 'normal'); + assert.deepEqual(p.tokens, ['island', 'basalt']); + assert.deepEqual(p.ignoredCommon, []); + assert.deepEqual(p.filesByToken.get('island'), + ['hot/cafe0001_p0.parquet', 'hot/cafe0001_p1.parquet']); +}); + +test('planQuery two-tier: all-common query → topk mode, single sidecar file', () => { const p = planQuery('material', MANIFEST); - assert.deepEqual(p.files, [ - 'hot/deadbeef_p0.parquet', 'hot/deadbeef_p1.parquet', 'hot/deadbeef_p2.parquet', - ]); + assert.equal(p.mode, 'topk'); + assert.deepEqual(p.tokens, ['material']); + assert.deepEqual(p.files, ['hot_topk.parquet']); +}); + +test('planQuery: expectedBytes = base shard sizes + fetchable-hot totals', () => { + const potteryShard = `shard_${String(fnv1a32('pottery') % 256).padStart(3, '0')}.parquet`; + const basaltShard = `shard_${String(fnv1a32('basalt') % 256).padStart(3, '0')}.parquet`; + const sizes = { [potteryShard]: 3_000_000, [basaltShard]: 2_000_000 }; + const p = planQuery('island pottery basalt material', MANIFEST, sizes); + // island (fetchable hot, 2.5MB) + pottery shard (3MB) + basalt shard (2MB); + // 'material' is dropped (common) and costs nothing. + assert.equal(p.expectedBytes, 2_500_000 + 3_000_000 + 2_000_000); + assert.deepEqual(p.ignoredCommon, ['material']); }); test('planQuery: shared shard between tokens is fetched once', () => { - // Construct two tokens that collide mod 2. const m = { shardCount: 1, hotTokens: {} }; - const p = planQuery('pottery basalt', m); + const p = planQuery('pottery basalt', m, { 'shard_000.parquet': 1000 }); assert.deepEqual(p.files, ['shard_000.parquet']); - assert.equal(p.filesByToken.get('pottery')[0], 'shard_000.parquet'); + assert.equal(p.expectedBytes, 1000); // counted once, not twice }); -test('bm25Contribution: hand-computed value', () => { - // df=10 of 1000 docs, tf=2, doc_len=4, avg=8, field weight 3.0 +const STATS = { + totalDocs: 1000, + avgDocLenByField: { + 'sample.label': 8, 'sample.description': 40, + 'sample.place_name': 8, 'concept.label': 8, + }, +}; + +test('bm25Contribution: hand-computed value (per-field avg doc len)', () => { const row = { field: 'sample.label', tf: 2, doc_len: 4, df: 10 }; - const stats = { totalDocs: 1000, avgDocLen: 8 }; const idf = Math.log((1000 - 10 + 0.5) / (10 + 0.5) + 1); const norm = (2 * 2.2) / (2 + 1.2 * (1 - 0.75 + 0.75 * 4 / 8)); - assert.ok(Math.abs(bm25Contribution(row, stats) - 3.0 * idf * norm) < 1e-12); + assert.ok(Math.abs(bm25Contribution(row, STATS) - 3.0 * idf * norm) < 1e-12); }); -test('bm25Contribution: rarer token scores higher, heavier field scores higher', () => { - const stats = { totalDocs: 1000, avgDocLen: 8 }; - const base = { field: 'sample.description', tf: 1, doc_len: 8 }; - const rare = bm25Contribution({ ...base, df: 2 }, stats); - const common = bm25Contribution({ ...base, df: 500 }, stats); +test('bm25Contribution: rarer token > common; field weights scale', () => { + const base = { field: 'sample.description', tf: 1, doc_len: 40 }; + const rare = bm25Contribution({ ...base, df: 2 }, STATS); + const common = bm25Contribution({ ...base, df: 500 }, STATS); assert.ok(rare > common); - const label = bm25Contribution({ ...base, df: 10, field: 'sample.label' }, stats); - const desc = bm25Contribution({ ...base, df: 10 }, stats); - assert.ok(Math.abs(label / desc - 3.0) < 1e-9); // weight ratio 3.0 : 1.0 + const label = bm25Contribution({ field: 'sample.label', tf: 1, doc_len: 8, df: 10 }, STATS); + const desc = bm25Contribution({ field: 'sample.description', tf: 1, doc_len: 40, df: 10 }, STATS); + // both rows sit exactly at their field's average length → norm equal; + // ratio is purely the field-weight ratio 3.0 : 1.0 + assert.ok(Math.abs(label / desc - 3.0) < 1e-9); }); -test('combineAndRank: AND semantics + score summation + stable ties', () => { - const stats = { totalDocs: 1000, avgDocLen: 8 }; +test('combineAndRank: AND semantics + per-pid summation + stable ties', () => { const row = (pid, field, df) => ({ pid, field, tf: 1, doc_len: 8, df }); const postings = new Map([ ['pottery', [row('p1', 'sample.label', 10), row('p2', 'sample.label', 10), row('p3', 'concept.label', 10)]], - ['cyprus', [row('p1', 'sample.description', 20), row('p3', 'sample.place_name', 20)]], + ['cyprus', [row('p1', 'sample.place_name', 20), row('p3', 'sample.place_name', 20)]], ]); - const out = combineAndRank(postings, stats); - // p2 matched only 'pottery' → excluded (AND). - assert.deepEqual(out.map(r => r.pid).sort(), ['p1', 'p3']); - // p1: label(3.0) + description(1.0); p3: concept(2.5) + place(2.0) → p3 wins. - assert.equal(out[0].pid, 'p3'); - // Ties break by pid ascending. + const out = combineAndRank(postings, STATS); + assert.deepEqual(out.map(r => r.pid).sort(), ['p1', 'p3']); // p2 fails AND + // p1: label(3.0)+place(2.0); p3: concept(2.5)+place(2.0) → p1 wins + assert.equal(out[0].pid, 'p1'); const tied = combineAndRank(new Map([ ['x', [row('b', 'sample.label', 10), row('a', 'sample.label', 10)]], - ]), stats); + ]), STATS); assert.deepEqual(tied.map(r => r.pid), ['a', 'b']); }); test('combineAndRank: top-K cap', () => { - const stats = { totalDocs: 1000, avgDocLen: 8 }; const rows = Array.from({ length: 80 }, (_, i) => ({ pid: `p${String(i).padStart(2, '0')}`, field: 'sample.label', tf: 1, doc_len: 8, df: 10 })); - const out = combineAndRank(new Map([['t', rows]]), stats); + const out = combineAndRank(new Map([['t', rows]]), STATS); assert.equal(out.length, 50); }); From 0db2cbc176380152ead1d3c187636c92596a25e9 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 17:54:51 -0700 Subject: [PATCH 3/7] #171 Phase B: ?fts=v1 substrate search path in the explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature-flagged wiring of the published sharded index (data.isamples.org/isamples_202608_search_index_v1/, PR #329): - buildSearchFilterSubstrate(term): plans via the unit-tested JS module (tokenize → stopwords → two-tier hot policy → file resolution), then scores in DuckDB SQL (BM25 §5, identical constants to the builder's hot_topk and the module's bm25Contribution; df embedded in rows — the query path fetches ONLY the plan's files). Produces the SAME singleton search_pids table (pid, label, source, place_name, relevance_score) as the interim path, so the table/points/facet-count semi-joins and the side panel are shared unchanged. - topk mode: all-common queries intersect the precomputed per-token top-500 sidecar. - Controlled-empty states for empty/all-stopword queries (§3 copy). - Three KB-scale sidecars (hot_tokens/shard_sizes/build_stats) fetched once on FIRST substrate search — flag-off visitors pay zero. - Disclosure (§3/§6): the results heading lists exactly which very common words were ignored ('material', 'sample', …) — silent semantic change is the one thing the policy forbids. - Route: doSearch's single buildSearchFilter call site branches on ?fts=v1. Default path byte-identical in behavior. Refs #171. Benchmark harness (the #172 input) is the remaining piece. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- explorer.qmd | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 3 deletions(-) diff --git a/explorer.qmd b/explorer.qmd index 62571e3..245b785 100644 --- a/explorer.qmd +++ b/explorer.qmd @@ -924,6 +924,10 @@ looksLikePid = _sqlBuilders.looksLikePid pidSearchWhere = _sqlBuilders.pidSearchWhere _explorerUtils = await import(new URL('assets/js/explorer-utils.js', document.baseURI).href) +// #171: substrate query engine (feature-flagged ?fts=v1 — see +// buildSearchFilterSubstrate). Pure planning/scoring logic lives in the +// module (Node-unit-tested); the explorer only wires it to DuckDB. +_searchSubstrate = await import(new URL('assets/js/search_substrate.js', document.baseURI).href) escapeHtml = _explorerUtils.escapeHtml facetCountsDisplayState = _explorerUtils.facetCountsDisplayState searchTerms = _explorerUtils.searchTerms @@ -5740,6 +5744,148 @@ zoomWatcher = { } } + // ===== #171: substrate search path (feature flag ?fts=v1) ===== + // Replaces buildSearchFilter's 69 MB facets ILIKE scan with the + // sharded inverted index at data.isamples.org/isamples_202608_search_index_v1/ + // (built by tools/build_search_index.py, contract SEARCH_INDEX_V1.md). + // Everything downstream is UNCHANGED: this produces the same singleton + // `search_pids` table (pid, label, source, place_name, relevance_score), + // so the table/points/facet-count semi-joins and the side panel work + // exactly as with the interim path. Opt-in only until the #172 gate. + const SUBSTRATE_BASE = `${R2_BASE}/isamples_202608_search_index_v1`; + let _substrateMeta = null; + + function substrateEnabled() { + try { + return new URLSearchParams(location.search).get('fts') === 'v1'; + } catch (e) { return false; } + } + + async function loadSubstrateMeta() { + // Three KB-scale sidecars, fetched once per session on FIRST substrate + // search (deliberately NOT at boot — flag-off visitors pay nothing). + if (_substrateMeta) return _substrateMeta; + const [hot, sizes, stats] = await Promise.all([ + fetch(`${SUBSTRATE_BASE}/hot_tokens.json`).then(r => r.json()), + fetch(`${SUBSTRATE_BASE}/shard_sizes.json`).then(r => r.json()), + fetch(`${SUBSTRATE_BASE}/build_stats.json`).then(r => r.json()), + ]); + _substrateMeta = { + manifest: { shardCount: stats.shard_count, hotTokens: hot.tokens }, + shardSizes: sizes, + stats: { + totalDocs: stats.total_documents, + avgDocLenByField: Object.fromEntries( + Object.entries(stats.fields).map(([f, v]) => [f, v.avg_doc_len])), + }, + }; + return _substrateMeta; + } + + async function buildSearchFilterSubstrate(term) { + const token = ++_searchFilterToken; + window.a1dbg?.('substrate-search-start', { term, token }); + const meta = await loadSubstrateMeta(); + const plan = _searchSubstrate.planQuery(term, meta.manifest, meta.shardSizes); + if (plan.mode === 'empty' || plan.mode === 'allStopwords') { + // Controlled empty per contract §3 — never a corpus dump. + await clearSearchFilter(); + window.__searchFilter = { + active: false, term: null, token: _searchFilterToken, total: 0, + kind: 'text', substrate: true, + note: plan.mode === 'allStopwords' + ? 'All terms in your query are common words. Add a more specific term to search.' + : null, + }; + return false; + } + // Tokens are lowercase alphanumeric BY TOKENIZER CONSTRUCTION (see + // search_tokenizer §4: every non-\p{L}\p{N} char became a separator), + // so interpolating them into SQL string literals cannot break out. + const tokList = plan.tokens.map(t => `'${t}'`).join(','); + const stats = meta.stats; + const staging = `search_pids_next_${token}`; + try { + if (plan.mode === 'topk') { + // All surviving terms are common: intersect the precomputed + // per-token top-500 lists, sum static scores (§6 — + // documented approximation, benchmarked at #172). + await db.query(` + CREATE OR REPLACE TABLE ${staging} AS + WITH agg AS ( + SELECT pid, sum(static_score) AS relevance_score, + count(DISTINCT token) AS m + FROM read_parquet('${SUBSTRATE_BASE}/hot_topk.parquet') + WHERE token IN (${tokList}) + GROUP BY pid + ) + SELECT a.pid, l.label, l.source, + CAST(l.place_name AS VARCHAR) AS place_name, + a.relevance_score + FROM agg a + LEFT JOIN read_parquet('${lite_url}') l USING (pid) + WHERE a.m = ${plan.tokens.length} + `); + } else { + const urls = plan.files.map(f => `'${SUBSTRATE_BASE}/${f}'`).join(','); + const avgCase = Object.entries(stats.avgDocLenByField) + .map(([f, v]) => `WHEN '${f}' THEN ${Number(v)}`).join(' '); + // BM25 per contract §5 — identical constants/shape to the + // builder's hot_topk scoring and the JS module's + // bm25Contribution (which unit-tests pin). df is EMBEDDED in + // the shard rows (round-5 amendment) — no df.parquet fetch. + await db.query(` + CREATE OR REPLACE TABLE ${staging} AS + WITH hits AS ( + SELECT token, pid, field, tf, doc_len, df + FROM read_parquet([${urls}]) + WHERE token IN (${tokList}) + ), contrib AS ( + SELECT token, pid, + (CASE field + WHEN 'sample.label' THEN 3.0 + WHEN 'concept.label' THEN 2.5 + WHEN 'sample.place_name' THEN 2.0 + ELSE 1.0 END) + * ln(((${stats.totalDocs} - df + 0.5) / (df + 0.5)) + 1) + * (tf * 2.2) / (tf + 1.2 * (0.25 + 0.75 * doc_len / + (CASE field ${avgCase} ELSE 10.0 END))) AS c + FROM hits + ), agg AS ( + SELECT pid, sum(c) AS relevance_score, + count(DISTINCT token) AS m + FROM contrib GROUP BY pid + ) + SELECT a.pid, l.label, l.source, + CAST(l.place_name AS VARCHAR) AS place_name, + a.relevance_score + FROM agg a + LEFT JOIN read_parquet('${lite_url}') l USING (pid) + WHERE a.m = ${plan.tokens.length} + `); + } + if (token !== _searchFilterToken) return false; // superseded + await db.query(`CREATE OR REPLACE TABLE search_pids AS + SELECT pid, label, source, place_name, relevance_score FROM ${staging}`); + if (token !== _searchFilterToken) return false; + const cnt = Array.from(await db.query(`SELECT COUNT(*) AS n FROM search_pids`)); + const total = cnt.length ? Number(cnt[0].n) : 0; + if (token !== _searchFilterToken) return false; + window.__searchFilter = { + active: true, term, token, total, kind: 'text', substrate: true, + substrateMode: plan.mode, + ignoredCommon: plan.ignoredCommon, + expectedBytes: plan.expectedBytes, + }; + window.a1dbg?.('substrate-search-end', + { term, token, total, mode: plan.mode, ignored: plan.ignoredCommon }); + return true; + } finally { + try { await db.query(`DROP TABLE IF EXISTS ${staging}`); } catch (e) { /* best effort */ } + } + } + // ===== end #171 substrate path ===== + async function clearSearchFilter() { _searchFilterToken++; window.__searchFilter = { active: false, term: null, token: _searchFilterToken, total: 0 }; @@ -5836,8 +5982,17 @@ zoomWatcher = { // reflects the committed search the table meta points at ("…shown in // the panel →"). z-index keeps it above scrolled rows. term is the // user's own input, rendered as-is to match the prior heading. - const searchHeadingHTML = (suffix) => - `

Search results: "${term}"${suffix}

`; + const searchHeadingHTML = (suffix) => { + // #171 disclosure (contract §3/§6): when the substrate path + // dropped common terms from the AND, the user must see exactly + // which words were ignored — silent semantic change is the one + // thing the policy forbids. + const sf = (typeof window !== 'undefined') ? window.__searchFilter : null; + const ignored = (sf && sf.substrate && sf.ignoredCommon && sf.ignoredCommon.length) + ? `
Very common word${sf.ignoredCommon.length > 1 ? 's' : ''} ignored: ${sf.ignoredCommon.map(escapeHtml).join(', ')}
` + : ''; + return `

Search results: "${term}"${suffix}${ignored}

`; + }; searchResults.textContent = effectiveScope === 'area' ? 'Searching selected areas...' : 'Searching entire world...'; @@ -5869,7 +6024,14 @@ zoomWatcher = { // empty-results branch say "Search error" rather than "No results". let searchFilterBuildFailed = false; try { - await buildSearchFilter(terms, term); + // #171: ?fts=v1 routes to the sharded-substrate path; both + // producers emit the identical `search_pids` singleton, so + // everything downstream is shared. + if (substrateEnabled()) { + await buildSearchFilterSubstrate(term); + } else { + await buildSearchFilter(terms, term); + } } catch (e) { console.warn('A1 search-filter build failed; surfaces stay unfiltered:', e); searchFilterBuildFailed = true; From 8bbb8806a4eec7fdc05e13bebddfeab15a58583a Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 18:02:05 -0700 Subject: [PATCH 4/7] #171: ship search_substrate.js + search_tokenizer.js as Quarto resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without these, the page-runtime import 404s and the OJS dependency cascade kills the whole search cell — boot green, search dead, smoke times out at the search step with no pageerror. (First page-runtime consumer of these modules; #170 only needed them for Node tests.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- _quarto.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/_quarto.yml b/_quarto.yml index 7bafcc4..adfe6c9 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -5,6 +5,13 @@ project: - assets/js/source-palette.js - assets/js/sql-builders.js - assets/js/explorer-utils.js + # #171: the substrate search engine + tokenizer are ES modules imported + # by explorer.qmd at page runtime — without these resource entries the + # import 404s and the OJS dependency cascade silently kills the entire + # search cell (boot fine, search dead: exactly what the smoke's 150s + # search timeout looks like). + - assets/js/search_substrate.js + - assets/js/search_tokenizer.js # #295: quarto.js unconditionally fetches /listings.json on every page # whose frontmatter declares `categories:` (explorer.qmd and others do, # for the tag chips under the title). Quarto only ever GENERATES that From 973f77d635f7bc792a4ff957533cf19d31096129 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 18:23:56 -0700 Subject: [PATCH 5/7] #171: end-to-end spec for the ?fts=v1 substrate path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six checks against the published index over the real CDN: ILIKE ground-truth parity (284), keyword-concept recall (pottery Cyprus 1,305, expectedBytes < 6 MB), common-term drop with user-visible disclosure, all-common topk mode engagement, controlled all-stopword empty, and default-path non-regression (interim search still used without the flag — the one test needing a raised timeout, because the interim path cold-downloads 69 MB while every substrate test fits the 30 s default; that asymmetry is the point). 6/6 green against the fork preview. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- tests/playwright/fts-v1.spec.js | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/playwright/fts-v1.spec.js diff --git a/tests/playwright/fts-v1.spec.js b/tests/playwright/fts-v1.spec.js new file mode 100644 index 0000000..e0134f9 --- /dev/null +++ b/tests/playwright/fts-v1.spec.js @@ -0,0 +1,94 @@ +// #171: substrate search path (?fts=v1) — end-to-end against the published +// index at data.isamples.org/isamples_202608_search_index_v1/. +// +// Run: BASE_URL=https://rdhyee.github.io/isamplesorg.github.io \ +// npx playwright test tests/playwright/fts-v1.spec.js +// +// Ground truths are properties of the 202608 index (see PR #329's +// committed build stats): 'axial seamount summit caldera' → 284; +// 'pottery Cyprus' → 1,305; 'basalt' alone → 785. + +const { test, expect } = require('@playwright/test'); + +const BASE = process.env.BASE_URL || 'https://rdhyee.github.io/isamplesorg.github.io'; +const URL = `${BASE}/explorer.html?fts=v1`; + +async function bootAndSearch(page, term) { + await page.fill('#sampleSearch', term); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction( + (t) => window.__searchFilter && window.__searchFilter.term === t + && window.__searchFilter.substrate === true, + term, { timeout: 120_000 }); + return page.evaluate(() => ({ + total: window.__searchFilter.total, + mode: window.__searchFilter.substrateMode, + ignored: window.__searchFilter.ignoredCommon || [], + expectedBytes: window.__searchFilter.expectedBytes, + })); +} + +test.describe('substrate search (?fts=v1)', () => { + test.beforeEach(async ({ page }) => { + await page.goto(URL, { timeout: 90_000 }); + await page.waitForSelector('.samples-table tbody tr[data-pid]', { timeout: 120_000 }); + }); + + test('multi-term place query matches ILIKE ground truth', async ({ page }) => { + const r = await bootAndSearch(page, 'axial seamount summit caldera'); + expect(r.mode).toBe('normal'); + expect(r.total).toBe(284); + }); + + test('keyword-concept recall: pottery Cyprus (the #326-class case)', async ({ page }) => { + const r = await bootAndSearch(page, 'pottery Cyprus'); + expect(r.mode).toBe('normal'); + expect(r.total).toBe(1305); + expect(r.expectedBytes).toBeGreaterThan(0); + expect(r.expectedBytes).toBeLessThan(6_000_000); + }); + + test('common term dropped from AND with disclosure', async ({ page }) => { + const r = await bootAndSearch(page, 'material basalt'); + expect(r.mode).toBe('normal'); + expect(r.ignored).toEqual(['material']); + expect(r.total).toBe(785); // == 'basalt' alone + // The §3/§6 disclosure must be user-visible in the results heading. + await page.waitForFunction(() => + /common word/i.test(document.getElementById('samplesSection')?.textContent || ''), + null, { timeout: 60_000 }); + }); + + test('all-common query engages topk mode (approximation, not error)', async ({ page }) => { + const r = await bootAndSearch(page, 'material sample'); + expect(r.mode).toBe('topk'); + // total may legitimately be small or 0 (top-500 intersection) — the + // assertion is that the MODE engaged and nothing crashed. #172's + // benchmark quantifies the empty-rate; union-rank is the candidate fix. + }); + + test('all-stopword query → controlled empty', async ({ page }) => { + await page.fill('#sampleSearch', 'a the of'); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction(() => + window.__searchFilter && window.__searchFilter.substrate === true + && window.__searchFilter.active === false, null, { timeout: 60_000 }); + const note = await page.evaluate(() => window.__searchFilter.note || ''); + expect(note).toMatch(/common words/i); + }); + + test('default path untouched: no fts param → interim search', async ({ page }) => { + // The INTERIM path cold-downloads the 69 MB facets parquet — the one + // test here that can't fit the 30 s config default. (The five substrate + // tests above all do; that asymmetry is the point of #171.) + test.setTimeout(240_000); + await page.goto(`${BASE}/explorer.html`, { timeout: 90_000 }); + await page.waitForSelector('.samples-table tbody tr[data-pid]', { timeout: 120_000 }); + await page.fill('#sampleSearch', 'basalt'); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction(() => + window.__searchFilter && window.__searchFilter.active, null, { timeout: 150_000 }); + const substrate = await page.evaluate(() => window.__searchFilter.substrate); + expect(substrate).toBeFalsy(); + }); +}); From 98824291850218b02a8a783943f907a64cb82519 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 18:28:52 -0700 Subject: [PATCH 6/7] =?UTF-8?q?#171:=20address=20Codex=20review=20?= =?UTF-8?q?=E2=80=94=20race=20guards,=20visible=20empty-copy,=20lazy=20imp?= =?UTF-8?q?ort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Superseded-search races: token guards added after the module-import and meta-fetch awaits, and before the controlled-empty clearSearchFilter (which bumps the token — a stale search calling it would invalidate the newer one). doSearch's build-failure catch now guards its clearSearchFilter on searchId === _searchSeq — an error inside a SUPERSEDED build must not blow away the newer search's filter (applies to both paths; strictly safer for the interim one). 2. All-stopword copy is now user-visible: the zero-results branch renders the substrate's controlled-empty note ("All terms in your query are common words…") in both the status line and the panel, instead of a generic "No results". Spec asserts the VISIBLE copy, not just internal state. 3. Flag-off isolation: the substrate module import moved from an OJS top-level cell (which flag-off visitors downloaded and whose failure could kill the whole search cell) to a lazy import inside buildSearchFilterSubstrate — first flagged search pays it, nobody else does, and a load failure surfaces as a caught search error. 110/110 unit tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- explorer.qmd | 30 ++++++++++++++++++++++-------- tests/playwright/fts-v1.spec.js | 5 +++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/explorer.qmd b/explorer.qmd index 245b785..27f70fd 100644 --- a/explorer.qmd +++ b/explorer.qmd @@ -924,10 +924,6 @@ looksLikePid = _sqlBuilders.looksLikePid pidSearchWhere = _sqlBuilders.pidSearchWhere _explorerUtils = await import(new URL('assets/js/explorer-utils.js', document.baseURI).href) -// #171: substrate query engine (feature-flagged ?fts=v1 — see -// buildSearchFilterSubstrate). Pure planning/scoring logic lives in the -// module (Node-unit-tested); the explorer only wires it to DuckDB. -_searchSubstrate = await import(new URL('assets/js/search_substrate.js', document.baseURI).href) escapeHtml = _explorerUtils.escapeHtml facetCountsDisplayState = _explorerUtils.facetCountsDisplayState searchTerms = _explorerUtils.searchTerms @@ -5782,10 +5778,20 @@ zoomWatcher = { return _substrateMeta; } + let _substrateModPromise = null; + async function buildSearchFilterSubstrate(term) { const token = ++_searchFilterToken; window.a1dbg?.('substrate-search-start', { term, token }); + // Lazy module import (Codex review): flag-off visitors must not + // download/evaluate the substrate modules, and a module-load failure + // must not be able to kill the default search cell — so the import + // happens HERE, on first flagged search, not as an OJS dependency. + _substrateModPromise ??= import(new URL('assets/js/search_substrate.js', document.baseURI).href); + const _searchSubstrate = await _substrateModPromise; + if (token !== _searchFilterToken) return false; // superseded during import const meta = await loadSubstrateMeta(); + if (token !== _searchFilterToken) return false; // superseded during meta fetch const plan = _searchSubstrate.planQuery(term, meta.manifest, meta.shardSizes); if (plan.mode === 'empty' || plan.mode === 'allStopwords') { // Controlled empty per contract §3 — never a corpus dump. @@ -6035,7 +6041,10 @@ zoomWatcher = { } catch (e) { console.warn('A1 search-filter build failed; surfaces stay unfiltered:', e); searchFilterBuildFailed = true; - await clearSearchFilter(); + // Guard the clear (Codex review of #171): an error inside a + // SUPERSEDED search's build must not blow away the newer + // search's freshly-built filter. Applies to both paths. + if (searchId === _searchSeq) await clearSearchFilter(); } // Superseded by a newer search while building? Bail before mutating UI. // Otherwise push the freshly-built filter through every surface: table, @@ -6192,17 +6201,22 @@ zoomWatcher = { resultsCount = results.length; if (results.length === 0) { // A build failure also empties search_pids → 0 rows here, so - // distinguish a genuine error from a true empty result set. + // distinguish a genuine error from a true empty result set — + // and (#171, Codex review) a substrate controlled-empty state + // (all-stopword query) must show ITS copy, not a generic + // "No results": the message tells the user what to do. + const substrateNote = (window.__searchFilter?.substrate && window.__searchFilter?.note) + ? window.__searchFilter.note : null; searchResults.textContent = searchFilterBuildFailed ? `Search error: couldn't build the filter for "${term}". Please try again.` - : `No results for "${term}"`; + : (substrateNote || `No results for "${term}"`); // Honesty fix (#247): the table meta points "→ panel", so the // panel must reflect THIS (empty) search rather than whatever // it showed before. Without this, a zero-result search left // stale prior content under a pointer claiming otherwise. const sampEl0 = document.getElementById('samplesSection'); if (sampEl0) sampEl0.innerHTML = searchHeadingHTML(searchFilterBuildFailed ? '' : ' (0)') - + `
${searchFilterBuildFailed ? 'Search failed to build — please try again.' : 'No samples matched this search.'}
`; + + `
${searchFilterBuildFailed ? 'Search failed to build — please try again.' : escapeHtml(substrateNote || 'No samples matched this search.')}
`; return; } diff --git a/tests/playwright/fts-v1.spec.js b/tests/playwright/fts-v1.spec.js index e0134f9..ed28ca1 100644 --- a/tests/playwright/fts-v1.spec.js +++ b/tests/playwright/fts-v1.spec.js @@ -75,6 +75,11 @@ test.describe('substrate search (?fts=v1)', () => { && window.__searchFilter.active === false, null, { timeout: 60_000 }); const note = await page.evaluate(() => window.__searchFilter.note || ''); expect(note).toMatch(/common words/i); + // The copy must be USER-VISIBLE (Codex review), not just internal state. + await page.waitForFunction(() => + /common words/i.test(document.getElementById('searchResults')?.textContent || '') + || /common words/i.test(document.getElementById('samplesSection')?.textContent || ''), + null, { timeout: 60_000 }); }); test('default path untouched: no fts param → interim search', async ({ page }) => { From 15fd2ce73892fa0649c7abcaa58bb419003c3e52 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 10 Jul 2026 18:43:33 -0700 Subject: [PATCH 7/7] #171: honest 120s suite timeout for cold-CDN e2e spec First substrate search pays boot + lazy import + 3 sidecars + shards; 22s observed passing, a cold CDN edge tips the 30s default. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AjnkWb4HpuLeDbYfzmY3X9 --- tests/playwright/fts-v1.spec.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/playwright/fts-v1.spec.js b/tests/playwright/fts-v1.spec.js index ed28ca1..b18a0f0 100644 --- a/tests/playwright/fts-v1.spec.js +++ b/tests/playwright/fts-v1.spec.js @@ -29,6 +29,11 @@ async function bootAndSearch(page, term) { } test.describe('substrate search (?fts=v1)', () => { + // Cold-CDN e2e: the first substrate search pays boot + lazy module import + // + three sidecar fetches + per-token shards. 22s was observed passing; + // a cold edge tips past the 30s config default, so the suite gets an + // honest budget (still far under the interim path's cold cost). + test.describe.configure({ timeout: 120_000 }); test.beforeEach(async ({ page }) => { await page.goto(URL, { timeout: 90_000 }); await page.waitForSelector('.samples-table tbody tr[data-pid]', { timeout: 120_000 });