diff --git a/_quarto.yml b/_quarto.yml index 7bafcc4..adfe6c9 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -5,6 +5,13 @@ project: - assets/js/source-palette.js - assets/js/sql-builders.js - assets/js/explorer-utils.js + # #171: the substrate search engine + tokenizer are ES modules imported + # by explorer.qmd at page runtime — without these resource entries the + # import 404s and the OJS dependency cascade silently kills the entire + # search cell (boot fine, search dead: exactly what the smoke's 150s + # search timeout looks like). + - assets/js/search_substrate.js + - assets/js/search_tokenizer.js # #295: quarto.js unconditionally fetches /listings.json on every page # whose frontmatter declares `categories:` (explorer.qmd and others do, # for the tag chips under the title). Quarto only ever GENERATES that diff --git a/assets/js/search_substrate.js b/assets/js/search_substrate.js new file mode 100644 index 0000000..b1bd7b6 --- /dev/null +++ b/assets/js/search_substrate.js @@ -0,0 +1,178 @@ +// Browser-side query engine for the v1 search substrate (#171, SEARCH_INDEX_V1.md). +// +// PURE functions only — no DuckDB, no fetch, no DOM. The explorer's flag +// path feeds this module data it fetched via db.query; the split keeps every +// piece of query logic unit-testable in Node +// (tests/unit/search-substrate.test.mjs) and the explorer wiring thin. +// +// Pipeline (contract §3, §5, §6 two-tier rule): +// tokenize (search_tokenizer.js) → drop query-time stopwords → dedupe → +// two-tier hot policy (fetchable hot joins the AND; non-fetchable common +// terms are dropped-with-disclosure, or an all-common query serves from +// the hot_topk sidecar) → resolve substrate files → BM25 per posting → +// field-weighted per-pid sums → AND across tokens → top-K. + +import { tokenize } from './search_tokenizer.js'; + +// FNV-1a 32-bit over UTF-8 bytes. MUST match tools/build_search_index.py's +// fnv1a32 exactly — parity pinned by tests/search_fnv1a_regression.json +// (values generated from the Python implementation). +export function fnv1a32(str) { + let h = 0x811c9dc5; + const bytes = new TextEncoder().encode(str); + for (const b of bytes) { + h ^= b; + // 32-bit multiply by the FNV prime 0x01000193 without BigInt: + h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0; + } + return h >>> 0; +} + +// Curated query-time stopword list (contract §3). Build-time indexes +// EVERYTHING; dropping happens here only, so the policy stays reversible. +export const QUERY_STOPWORDS = new Set([ + 'a', 'an', 'the', 'of', 'from', 'for', 'to', 'in', 'on', 'at', + 'is', 'was', 'with', 'and', 'or', +]); + +// Field weights (contract §5) + BM25 constants. +export const FIELD_WEIGHTS = { + 'sample.label': 3.0, + 'concept.label': 2.5, + 'sample.place_name': 2.0, + 'sample.description': 1.0, +}; +export const BM25_K1 = 1.2; +export const BM25_B = 0.75; +export const TOP_K = 50; + +const shardFile = (token, shardCount) => + `shard_${String(fnv1a32(token) % shardCount).padStart(3, '0')}.parquet`; + +/** + * Plan a query under the §6 two-tier hot rule. + * + * @param term raw user input + * @param manifest { shardCount, hotTokens } — shardCount from + * build_stats.json; hotTokens = hot_tokens.json's `tokens` + * ({token: {key, sub_files, postings, total_bytes, + * fetchable}}). + * @param shardSizes optional shard_sizes.json object ({file: bytes}); when + * given, the plan carries expectedBytes so callers can + * report transfer up-front (§7). + * @returns {{ + * mode: 'empty'|'allStopwords'|'normal'|'topk', + * tokens: string[], // tokens participating in the AND + * ignoredCommon: string[], // non-fetchable hot terms dropped (§3 — + * // the UI MUST disclose these) + * files: string[], // deduped relative substrate paths to fetch + * filesByToken: Map, + * expectedBytes: number|null, + * }} + */ +export function planQuery(term, manifest, shardSizes = null) { + const raw = tokenize(term); + const survivors = []; + for (const t of raw) { + if (QUERY_STOPWORDS.has(t)) continue; + if (!survivors.includes(t)) survivors.push(t); // duplicate-term dedup + } + const base = { ignoredCommon: [], files: [], filesByToken: new Map(), expectedBytes: null }; + if (raw.length === 0) return { ...base, mode: 'empty', tokens: [] }; + if (survivors.length === 0) return { ...base, mode: 'allStopwords', tokens: [] }; + + const hot = manifest.hotTokens || {}; + const participating = []; + const common = []; + for (const t of survivors) { + const h = hot[t]; + if (h && !h.fetchable) common.push(t); + else participating.push(t); + } + + if (participating.length === 0) { + // Every surviving term is a common term: rank via the sidecar. + return { + mode: 'topk', tokens: common, ignoredCommon: [], + files: ['hot_topk.parquet'], + filesByToken: new Map(common.map(t => [t, ['hot_topk.parquet']])), + expectedBytes: null, + }; + } + + const filesByToken = new Map(); + const files = []; + for (const t of participating) { + const h = hot[t]; + const tokenFiles = (h && h.fetchable) + ? Array.from({ length: h.sub_files }, + (_, m) => `hot/${h.key}_p${m}.parquet`) + : [shardFile(t, manifest.shardCount)]; + filesByToken.set(t, tokenFiles); + for (const f of tokenFiles) if (!files.includes(f)) files.push(f); + } + let expectedBytes = null; + if (shardSizes) { + expectedBytes = 0; + const counted = new Set(); + for (const t of participating) { + const h = hot[t]; + if (h && h.fetchable) { + if (!counted.has(h.key)) { expectedBytes += h.total_bytes; counted.add(h.key); } + } else { + const f = shardFile(t, manifest.shardCount); + if (!counted.has(f)) { expectedBytes += shardSizes[f] ?? 0; counted.add(f); } + } + } + } + return { + mode: 'normal', tokens: participating, ignoredCommon: common, + files, filesByToken, expectedBytes, + }; +} + +/** + * BM25 contribution of one posting row (contract §5). + * @param row { field, tf, doc_len, df } — df is EMBEDDED in shipped rows + * (round-5 amendment; df.parquet is offline-only). + * @param stats { totalDocs, avgDocLenByField } — from build_stats.json: + * totalDocs = total_documents (distinct (pid, field) docs); + * avgDocLenByField[field] = fields[field].avg_doc_len + * (PER-FIELD corpus averages, matching the builder's + * hot_topk scoring). + */ +export function bm25Contribution(row, stats) { + const idf = Math.log((stats.totalDocs - row.df + 0.5) / (row.df + 0.5) + 1); + const avgDl = stats.avgDocLenByField[row.field]; + const norm = row.tf * (BM25_K1 + 1) + / (row.tf + BM25_K1 * (1 - BM25_B + BM25_B * row.doc_len / avgDl)); + const weight = FIELD_WEIGHTS[row.field] ?? 1.0; + return weight * idf * norm; +} + +/** + * Combine postings into the final AND-ranked top-K. + * + * @param postingsByToken Map> + * @param stats { totalDocs, avgDocLenByField } + * @param k top-K cap (default contract TOP_K = 50) + * @returns Array<{pid, score}> sorted score desc, pid asc for stable ties. + */ +export function combineAndRank(postingsByToken, stats, k = TOP_K) { + const scores = new Map(); // pid -> summed score + const matched = new Map(); // pid -> Set + for (const [token, rows] of postingsByToken) { + for (const row of rows) { + scores.set(row.pid, (scores.get(row.pid) ?? 0) + bm25Contribution(row, stats)); + if (!matched.has(row.pid)) matched.set(row.pid, new Set()); + matched.get(row.pid).add(token); + } + } + const need = postingsByToken.size; // AND semantics: every token matches + const out = []; + for (const [pid, tokens] of matched) { + if (tokens.size === need) out.push({ pid, score: scores.get(pid) }); + } + out.sort((a, b) => (b.score - a.score) || (a.pid < b.pid ? -1 : 1)); + return out.slice(0, k); +} diff --git a/explorer.qmd b/explorer.qmd index 62571e3..27f70fd 100644 --- a/explorer.qmd +++ b/explorer.qmd @@ -5740,6 +5740,158 @@ zoomWatcher = { } } + // ===== #171: substrate search path (feature flag ?fts=v1) ===== + // Replaces buildSearchFilter's 69 MB facets ILIKE scan with the + // sharded inverted index at data.isamples.org/isamples_202608_search_index_v1/ + // (built by tools/build_search_index.py, contract SEARCH_INDEX_V1.md). + // Everything downstream is UNCHANGED: this produces the same singleton + // `search_pids` table (pid, label, source, place_name, relevance_score), + // so the table/points/facet-count semi-joins and the side panel work + // exactly as with the interim path. Opt-in only until the #172 gate. + const SUBSTRATE_BASE = `${R2_BASE}/isamples_202608_search_index_v1`; + let _substrateMeta = null; + + function substrateEnabled() { + try { + return new URLSearchParams(location.search).get('fts') === 'v1'; + } catch (e) { return false; } + } + + async function loadSubstrateMeta() { + // Three KB-scale sidecars, fetched once per session on FIRST substrate + // search (deliberately NOT at boot — flag-off visitors pay nothing). + if (_substrateMeta) return _substrateMeta; + const [hot, sizes, stats] = await Promise.all([ + fetch(`${SUBSTRATE_BASE}/hot_tokens.json`).then(r => r.json()), + fetch(`${SUBSTRATE_BASE}/shard_sizes.json`).then(r => r.json()), + fetch(`${SUBSTRATE_BASE}/build_stats.json`).then(r => r.json()), + ]); + _substrateMeta = { + manifest: { shardCount: stats.shard_count, hotTokens: hot.tokens }, + shardSizes: sizes, + stats: { + totalDocs: stats.total_documents, + avgDocLenByField: Object.fromEntries( + Object.entries(stats.fields).map(([f, v]) => [f, v.avg_doc_len])), + }, + }; + return _substrateMeta; + } + + let _substrateModPromise = null; + + async function buildSearchFilterSubstrate(term) { + const token = ++_searchFilterToken; + window.a1dbg?.('substrate-search-start', { term, token }); + // Lazy module import (Codex review): flag-off visitors must not + // download/evaluate the substrate modules, and a module-load failure + // must not be able to kill the default search cell — so the import + // happens HERE, on first flagged search, not as an OJS dependency. + _substrateModPromise ??= import(new URL('assets/js/search_substrate.js', document.baseURI).href); + const _searchSubstrate = await _substrateModPromise; + if (token !== _searchFilterToken) return false; // superseded during import + const meta = await loadSubstrateMeta(); + if (token !== _searchFilterToken) return false; // superseded during meta fetch + const plan = _searchSubstrate.planQuery(term, meta.manifest, meta.shardSizes); + if (plan.mode === 'empty' || plan.mode === 'allStopwords') { + // Controlled empty per contract §3 — never a corpus dump. + await clearSearchFilter(); + window.__searchFilter = { + active: false, term: null, token: _searchFilterToken, total: 0, + kind: 'text', substrate: true, + note: plan.mode === 'allStopwords' + ? 'All terms in your query are common words. Add a more specific term to search.' + : null, + }; + return false; + } + // Tokens are lowercase alphanumeric BY TOKENIZER CONSTRUCTION (see + // search_tokenizer §4: every non-\p{L}\p{N} char became a separator), + // so interpolating them into SQL string literals cannot break out. + const tokList = plan.tokens.map(t => `'${t}'`).join(','); + const stats = meta.stats; + const staging = `search_pids_next_${token}`; + try { + if (plan.mode === 'topk') { + // All surviving terms are common: intersect the precomputed + // per-token top-500 lists, sum static scores (§6 — + // documented approximation, benchmarked at #172). + await db.query(` + CREATE OR REPLACE TABLE ${staging} AS + WITH agg AS ( + SELECT pid, sum(static_score) AS relevance_score, + count(DISTINCT token) AS m + FROM read_parquet('${SUBSTRATE_BASE}/hot_topk.parquet') + WHERE token IN (${tokList}) + GROUP BY pid + ) + SELECT a.pid, l.label, l.source, + CAST(l.place_name AS VARCHAR) AS place_name, + a.relevance_score + FROM agg a + LEFT JOIN read_parquet('${lite_url}') l USING (pid) + WHERE a.m = ${plan.tokens.length} + `); + } else { + const urls = plan.files.map(f => `'${SUBSTRATE_BASE}/${f}'`).join(','); + const avgCase = Object.entries(stats.avgDocLenByField) + .map(([f, v]) => `WHEN '${f}' THEN ${Number(v)}`).join(' '); + // BM25 per contract §5 — identical constants/shape to the + // builder's hot_topk scoring and the JS module's + // bm25Contribution (which unit-tests pin). df is EMBEDDED in + // the shard rows (round-5 amendment) — no df.parquet fetch. + await db.query(` + CREATE OR REPLACE TABLE ${staging} AS + WITH hits AS ( + SELECT token, pid, field, tf, doc_len, df + FROM read_parquet([${urls}]) + WHERE token IN (${tokList}) + ), contrib AS ( + SELECT token, pid, + (CASE field + WHEN 'sample.label' THEN 3.0 + WHEN 'concept.label' THEN 2.5 + WHEN 'sample.place_name' THEN 2.0 + ELSE 1.0 END) + * ln(((${stats.totalDocs} - df + 0.5) / (df + 0.5)) + 1) + * (tf * 2.2) / (tf + 1.2 * (0.25 + 0.75 * doc_len / + (CASE field ${avgCase} ELSE 10.0 END))) AS c + FROM hits + ), agg AS ( + SELECT pid, sum(c) AS relevance_score, + count(DISTINCT token) AS m + FROM contrib GROUP BY pid + ) + SELECT a.pid, l.label, l.source, + CAST(l.place_name AS VARCHAR) AS place_name, + a.relevance_score + FROM agg a + LEFT JOIN read_parquet('${lite_url}') l USING (pid) + WHERE a.m = ${plan.tokens.length} + `); + } + if (token !== _searchFilterToken) return false; // superseded + await db.query(`CREATE OR REPLACE TABLE search_pids AS + SELECT pid, label, source, place_name, relevance_score FROM ${staging}`); + if (token !== _searchFilterToken) return false; + const cnt = Array.from(await db.query(`SELECT COUNT(*) AS n FROM search_pids`)); + const total = cnt.length ? Number(cnt[0].n) : 0; + if (token !== _searchFilterToken) return false; + window.__searchFilter = { + active: true, term, token, total, kind: 'text', substrate: true, + substrateMode: plan.mode, + ignoredCommon: plan.ignoredCommon, + expectedBytes: plan.expectedBytes, + }; + window.a1dbg?.('substrate-search-end', + { term, token, total, mode: plan.mode, ignored: plan.ignoredCommon }); + return true; + } finally { + try { await db.query(`DROP TABLE IF EXISTS ${staging}`); } catch (e) { /* best effort */ } + } + } + // ===== end #171 substrate path ===== + async function clearSearchFilter() { _searchFilterToken++; window.__searchFilter = { active: false, term: null, token: _searchFilterToken, total: 0 }; @@ -5836,8 +5988,17 @@ zoomWatcher = { // reflects the committed search the table meta points at ("…shown in // the panel →"). z-index keeps it above scrolled rows. term is the // user's own input, rendered as-is to match the prior heading. - const searchHeadingHTML = (suffix) => - `

Search results: "${term}"${suffix}

`; + const searchHeadingHTML = (suffix) => { + // #171 disclosure (contract §3/§6): when the substrate path + // dropped common terms from the AND, the user must see exactly + // which words were ignored — silent semantic change is the one + // thing the policy forbids. + const sf = (typeof window !== 'undefined') ? window.__searchFilter : null; + const ignored = (sf && sf.substrate && sf.ignoredCommon && sf.ignoredCommon.length) + ? `
Very common word${sf.ignoredCommon.length > 1 ? 's' : ''} ignored: ${sf.ignoredCommon.map(escapeHtml).join(', ')}
` + : ''; + return `

Search results: "${term}"${suffix}${ignored}

`; + }; searchResults.textContent = effectiveScope === 'area' ? 'Searching selected areas...' : 'Searching entire world...'; @@ -5869,11 +6030,21 @@ zoomWatcher = { // empty-results branch say "Search error" rather than "No results". let searchFilterBuildFailed = false; try { - await buildSearchFilter(terms, term); + // #171: ?fts=v1 routes to the sharded-substrate path; both + // producers emit the identical `search_pids` singleton, so + // everything downstream is shared. + if (substrateEnabled()) { + await buildSearchFilterSubstrate(term); + } else { + await buildSearchFilter(terms, term); + } } catch (e) { console.warn('A1 search-filter build failed; surfaces stay unfiltered:', e); searchFilterBuildFailed = true; - await clearSearchFilter(); + // Guard the clear (Codex review of #171): an error inside a + // SUPERSEDED search's build must not blow away the newer + // search's freshly-built filter. Applies to both paths. + if (searchId === _searchSeq) await clearSearchFilter(); } // Superseded by a newer search while building? Bail before mutating UI. // Otherwise push the freshly-built filter through every surface: table, @@ -6030,17 +6201,22 @@ zoomWatcher = { resultsCount = results.length; if (results.length === 0) { // A build failure also empties search_pids → 0 rows here, so - // distinguish a genuine error from a true empty result set. + // distinguish a genuine error from a true empty result set — + // and (#171, Codex review) a substrate controlled-empty state + // (all-stopword query) must show ITS copy, not a generic + // "No results": the message tells the user what to do. + const substrateNote = (window.__searchFilter?.substrate && window.__searchFilter?.note) + ? window.__searchFilter.note : null; searchResults.textContent = searchFilterBuildFailed ? `Search error: couldn't build the filter for "${term}". Please try again.` - : `No results for "${term}"`; + : (substrateNote || `No results for "${term}"`); // Honesty fix (#247): the table meta points "→ panel", so the // panel must reflect THIS (empty) search rather than whatever // it showed before. Without this, a zero-result search left // stale prior content under a pointer claiming otherwise. const sampEl0 = document.getElementById('samplesSection'); if (sampEl0) sampEl0.innerHTML = searchHeadingHTML(searchFilterBuildFailed ? '' : ' (0)') - + `
${searchFilterBuildFailed ? 'Search failed to build — please try again.' : 'No samples matched this search.'}
`; + + `
${searchFilterBuildFailed ? 'Search failed to build — please try again.' : escapeHtml(substrateNote || 'No samples matched this search.')}
`; return; } diff --git a/tests/playwright/fts-v1.spec.js b/tests/playwright/fts-v1.spec.js new file mode 100644 index 0000000..b18a0f0 --- /dev/null +++ b/tests/playwright/fts-v1.spec.js @@ -0,0 +1,104 @@ +// #171: substrate search path (?fts=v1) — end-to-end against the published +// index at data.isamples.org/isamples_202608_search_index_v1/. +// +// Run: BASE_URL=https://rdhyee.github.io/isamplesorg.github.io \ +// npx playwright test tests/playwright/fts-v1.spec.js +// +// Ground truths are properties of the 202608 index (see PR #329's +// committed build stats): 'axial seamount summit caldera' → 284; +// 'pottery Cyprus' → 1,305; 'basalt' alone → 785. + +const { test, expect } = require('@playwright/test'); + +const BASE = process.env.BASE_URL || 'https://rdhyee.github.io/isamplesorg.github.io'; +const URL = `${BASE}/explorer.html?fts=v1`; + +async function bootAndSearch(page, term) { + await page.fill('#sampleSearch', term); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction( + (t) => window.__searchFilter && window.__searchFilter.term === t + && window.__searchFilter.substrate === true, + term, { timeout: 120_000 }); + return page.evaluate(() => ({ + total: window.__searchFilter.total, + mode: window.__searchFilter.substrateMode, + ignored: window.__searchFilter.ignoredCommon || [], + expectedBytes: window.__searchFilter.expectedBytes, + })); +} + +test.describe('substrate search (?fts=v1)', () => { + // Cold-CDN e2e: the first substrate search pays boot + lazy module import + // + three sidecar fetches + per-token shards. 22s was observed passing; + // a cold edge tips past the 30s config default, so the suite gets an + // honest budget (still far under the interim path's cold cost). + test.describe.configure({ timeout: 120_000 }); + test.beforeEach(async ({ page }) => { + await page.goto(URL, { timeout: 90_000 }); + await page.waitForSelector('.samples-table tbody tr[data-pid]', { timeout: 120_000 }); + }); + + test('multi-term place query matches ILIKE ground truth', async ({ page }) => { + const r = await bootAndSearch(page, 'axial seamount summit caldera'); + expect(r.mode).toBe('normal'); + expect(r.total).toBe(284); + }); + + test('keyword-concept recall: pottery Cyprus (the #326-class case)', async ({ page }) => { + const r = await bootAndSearch(page, 'pottery Cyprus'); + expect(r.mode).toBe('normal'); + expect(r.total).toBe(1305); + expect(r.expectedBytes).toBeGreaterThan(0); + expect(r.expectedBytes).toBeLessThan(6_000_000); + }); + + test('common term dropped from AND with disclosure', async ({ page }) => { + const r = await bootAndSearch(page, 'material basalt'); + expect(r.mode).toBe('normal'); + expect(r.ignored).toEqual(['material']); + expect(r.total).toBe(785); // == 'basalt' alone + // The §3/§6 disclosure must be user-visible in the results heading. + await page.waitForFunction(() => + /common word/i.test(document.getElementById('samplesSection')?.textContent || ''), + null, { timeout: 60_000 }); + }); + + test('all-common query engages topk mode (approximation, not error)', async ({ page }) => { + const r = await bootAndSearch(page, 'material sample'); + expect(r.mode).toBe('topk'); + // total may legitimately be small or 0 (top-500 intersection) — the + // assertion is that the MODE engaged and nothing crashed. #172's + // benchmark quantifies the empty-rate; union-rank is the candidate fix. + }); + + test('all-stopword query → controlled empty', async ({ page }) => { + await page.fill('#sampleSearch', 'a the of'); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction(() => + window.__searchFilter && window.__searchFilter.substrate === true + && window.__searchFilter.active === false, null, { timeout: 60_000 }); + const note = await page.evaluate(() => window.__searchFilter.note || ''); + expect(note).toMatch(/common words/i); + // The copy must be USER-VISIBLE (Codex review), not just internal state. + await page.waitForFunction(() => + /common words/i.test(document.getElementById('searchResults')?.textContent || '') + || /common words/i.test(document.getElementById('samplesSection')?.textContent || ''), + null, { timeout: 60_000 }); + }); + + test('default path untouched: no fts param → interim search', async ({ page }) => { + // The INTERIM path cold-downloads the 69 MB facets parquet — the one + // test here that can't fit the 30 s config default. (The five substrate + // tests above all do; that asymmetry is the point of #171.) + test.setTimeout(240_000); + await page.goto(`${BASE}/explorer.html`, { timeout: 90_000 }); + await page.waitForSelector('.samples-table tbody tr[data-pid]', { timeout: 120_000 }); + await page.fill('#sampleSearch', 'basalt'); + await page.locator('#searchSubmitBtn').first().click(); + await page.waitForFunction(() => + window.__searchFilter && window.__searchFilter.active, null, { timeout: 150_000 }); + const substrate = await page.evaluate(() => window.__searchFilter.substrate); + expect(substrate).toBeFalsy(); + }); +}); diff --git a/tests/search_fnv1a_regression.json b/tests/search_fnv1a_regression.json new file mode 100644 index 0000000..d818159 --- /dev/null +++ b/tests/search_fnv1a_regression.json @@ -0,0 +1,82 @@ +[ + { + "input": "pottery", + "fnv1a32": 3121265474 + }, + { + "input": "cyprus", + "fnv1a32": 1194896787 + }, + { + "input": "basalt", + "fnv1a32": 3758865220 + }, + { + "input": "catalhoyuk", + "fnv1a32": 1680059526 + }, + { + "input": "material", + "fnv1a32": 3538210912 + }, + { + "input": "a", + "fnv1a32": 3826002220 + }, + { + "input": "z", + "fnv1a32": 4278997933 + }, + { + "input": "igsn", + "fnv1a32": 877601424 + }, + { + "input": "hrv000abc", + "fnv1a32": 4165183063 + }, + { + "input": "1965", + "fnv1a32": 676065338 + }, + { + "input": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "fnv1a32": 3516603077 + }, + { + "input": "陶器", + "fnv1a32": 1823698683 + }, + { + "input": "сосуд", + "fnv1a32": 3730779371 + }, + { + "input": "κεραμικος", + "fnv1a32": 2429063684 + }, + { + "input": "unknownstuff", + "fnv1a32": 1666634825 + }, + { + "input": "volcanic", + "fnv1a32": 1893227256 + }, + { + "input": "rock", + "fnv1a32": 974867124 + }, + { + "input": "seamount", + "fnv1a32": 2199515177 + }, + { + "input": "0", + "fnv1a32": 890022063 + }, + { + "input": "9999999", + "fnv1a32": 202886326 + } +] \ No newline at end of file diff --git a/tests/unit/search-substrate.test.mjs b/tests/unit/search-substrate.test.mjs new file mode 100644 index 0000000..9cfa52d --- /dev/null +++ b/tests/unit/search-substrate.test.mjs @@ -0,0 +1,163 @@ +// Unit tests for assets/js/search_substrate.js (#171). +// FNV-1a parity values come from tools/build_search_index.py (fixture +// generated by the Python implementation — same pattern as the tokenizer +// regression set). The manifest shapes mirror hot_tokens.json's two-tier +// schema (fetchable vs common terms, contract §6). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +import { + fnv1a32, planQuery, bm25Contribution, combineAndRank, + QUERY_STOPWORDS, FIELD_WEIGHTS, BM25_K1, BM25_B, +} from '../../assets/js/search_substrate.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const FNV = JSON.parse(readFileSync(join(here, '..', 'search_fnv1a_regression.json'), 'utf8')); + +test('fnv1a32 parity with Python (fixture incl. CJK/Cyrillic/Greek)', () => { + for (const { input, fnv1a32: expected } of FNV) { + assert.equal(fnv1a32(input), expected, `fnv1a32(${JSON.stringify(input)})`); + } +}); + +// Two-tier manifest: 'material' is a true common term (non-fetchable); +// 'island' is storage-hot but fetchable (joins the AND normally). +const MANIFEST = { + shardCount: 256, + hotTokens: { + material: { key: 'deadbeef', sub_files: 10, postings: 5_000_000, + total_bytes: 50_000_000, fetchable: false }, + island: { key: 'cafe0001', sub_files: 2, postings: 82_000, + total_bytes: 2_500_000, fetchable: true }, + }, +}; + +test('planQuery: stopword policy + duplicate-term dedup', () => { + const p1 = planQuery('pottery from Cyprus', MANIFEST); + assert.equal(p1.mode, 'normal'); + assert.deepEqual(p1.tokens, ['pottery', 'cyprus']); // 'from' dropped + const p2 = planQuery('pottery pottery cyprus', MANIFEST); + assert.deepEqual(p2.tokens, ['pottery', 'cyprus']); // dupes collapse + // #172 hard-fail precondition: both plans fetch identical files. + assert.deepEqual(p1.files, p2.files); +}); + +test('planQuery: all-stopword input → controlled empty, no fetches', () => { + const p = planQuery('a the of', MANIFEST); + assert.equal(p.mode, 'allStopwords'); + assert.deepEqual(p.files, []); +}); + +test('planQuery: empty/whitespace input', () => { + assert.equal(planQuery(' ', MANIFEST).mode, 'empty'); +}); + +test('planQuery: base-shard path uses fnv1a32 % shardCount, zero-padded', () => { + const p = planQuery('pottery', MANIFEST); + const shard = fnv1a32('pottery') % 256; + assert.deepEqual(p.files, [`shard_${String(shard).padStart(3, '0')}.parquet`]); +}); + +test('planQuery two-tier: common term dropped WITH disclosure when selective terms exist', () => { + const p = planQuery('material basalt', MANIFEST); + assert.equal(p.mode, 'normal'); + assert.deepEqual(p.tokens, ['basalt']); // AND excludes 'material' + assert.deepEqual(p.ignoredCommon, ['material']); // UI must disclose this + assert.ok(!p.files.some(f => f.startsWith('hot/'))); // no hot bytes fetched +}); + +test('planQuery two-tier: fetchable hot joins the AND via its sub-files', () => { + const p = planQuery('island basalt', MANIFEST); + assert.equal(p.mode, 'normal'); + assert.deepEqual(p.tokens, ['island', 'basalt']); + assert.deepEqual(p.ignoredCommon, []); + assert.deepEqual(p.filesByToken.get('island'), + ['hot/cafe0001_p0.parquet', 'hot/cafe0001_p1.parquet']); +}); + +test('planQuery two-tier: all-common query → topk mode, single sidecar file', () => { + const p = planQuery('material', MANIFEST); + assert.equal(p.mode, 'topk'); + assert.deepEqual(p.tokens, ['material']); + assert.deepEqual(p.files, ['hot_topk.parquet']); +}); + +test('planQuery: expectedBytes = base shard sizes + fetchable-hot totals', () => { + const potteryShard = `shard_${String(fnv1a32('pottery') % 256).padStart(3, '0')}.parquet`; + const basaltShard = `shard_${String(fnv1a32('basalt') % 256).padStart(3, '0')}.parquet`; + const sizes = { [potteryShard]: 3_000_000, [basaltShard]: 2_000_000 }; + const p = planQuery('island pottery basalt material', MANIFEST, sizes); + // island (fetchable hot, 2.5MB) + pottery shard (3MB) + basalt shard (2MB); + // 'material' is dropped (common) and costs nothing. + assert.equal(p.expectedBytes, 2_500_000 + 3_000_000 + 2_000_000); + assert.deepEqual(p.ignoredCommon, ['material']); +}); + +test('planQuery: shared shard between tokens is fetched once', () => { + const m = { shardCount: 1, hotTokens: {} }; + const p = planQuery('pottery basalt', m, { 'shard_000.parquet': 1000 }); + assert.deepEqual(p.files, ['shard_000.parquet']); + assert.equal(p.expectedBytes, 1000); // counted once, not twice +}); + +const STATS = { + totalDocs: 1000, + avgDocLenByField: { + 'sample.label': 8, 'sample.description': 40, + 'sample.place_name': 8, 'concept.label': 8, + }, +}; + +test('bm25Contribution: hand-computed value (per-field avg doc len)', () => { + const row = { field: 'sample.label', tf: 2, doc_len: 4, df: 10 }; + const idf = Math.log((1000 - 10 + 0.5) / (10 + 0.5) + 1); + const norm = (2 * 2.2) / (2 + 1.2 * (1 - 0.75 + 0.75 * 4 / 8)); + assert.ok(Math.abs(bm25Contribution(row, STATS) - 3.0 * idf * norm) < 1e-12); +}); + +test('bm25Contribution: rarer token > common; field weights scale', () => { + const base = { field: 'sample.description', tf: 1, doc_len: 40 }; + const rare = bm25Contribution({ ...base, df: 2 }, STATS); + const common = bm25Contribution({ ...base, df: 500 }, STATS); + assert.ok(rare > common); + const label = bm25Contribution({ field: 'sample.label', tf: 1, doc_len: 8, df: 10 }, STATS); + const desc = bm25Contribution({ field: 'sample.description', tf: 1, doc_len: 40, df: 10 }, STATS); + // both rows sit exactly at their field's average length → norm equal; + // ratio is purely the field-weight ratio 3.0 : 1.0 + assert.ok(Math.abs(label / desc - 3.0) < 1e-9); +}); + +test('combineAndRank: AND semantics + per-pid summation + stable ties', () => { + const row = (pid, field, df) => ({ pid, field, tf: 1, doc_len: 8, df }); + const postings = new Map([ + ['pottery', [row('p1', 'sample.label', 10), row('p2', 'sample.label', 10), + row('p3', 'concept.label', 10)]], + ['cyprus', [row('p1', 'sample.place_name', 20), row('p3', 'sample.place_name', 20)]], + ]); + const out = combineAndRank(postings, STATS); + assert.deepEqual(out.map(r => r.pid).sort(), ['p1', 'p3']); // p2 fails AND + // p1: label(3.0)+place(2.0); p3: concept(2.5)+place(2.0) → p1 wins + assert.equal(out[0].pid, 'p1'); + const tied = combineAndRank(new Map([ + ['x', [row('b', 'sample.label', 10), row('a', 'sample.label', 10)]], + ]), STATS); + assert.deepEqual(tied.map(r => r.pid), ['a', 'b']); +}); + +test('combineAndRank: top-K cap', () => { + const rows = Array.from({ length: 80 }, (_, i) => + ({ pid: `p${String(i).padStart(2, '0')}`, field: 'sample.label', + tf: 1, doc_len: 8, df: 10 })); + const out = combineAndRank(new Map([['t', rows]]), STATS); + assert.equal(out.length, 50); +}); + +test('constants match the contract', () => { + assert.equal(BM25_K1, 1.2); + assert.equal(BM25_B, 0.75); + assert.deepEqual(FIELD_WEIGHTS['sample.label'], 3.0); + assert.ok(QUERY_STOPWORDS.has('from') && !QUERY_STOPWORDS.has('pottery')); +});