Skip to content

#171: defer the lite join off the substrate filter-build critical path #333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdhyee merged 2 commits into from
Jul 11, 2026
Merged

#171: defer the lite join off the substrate filter-build critical path #333

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ddd7215
#171: defer the lite join out of the substrate filter-build critical …
rdhyee Jul 11, 2026
5f14738
#171: keep the cheap 50-row world-panel shape except substrate+source…
rdhyee Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 56 additions & 14 deletions explorer.qmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5836,11 +5836,11 @@ zoomWatcher = {
WHERE token IN (${tokList})
GROUP BY pid
)
SELECT a.pid, l.label, l.source,
CAST(l.place_name AS VARCHAR) AS place_name,
SELECT a.pid,
NULL::VARCHAR AS label, NULL::VARCHAR AS source,
NULL::VARCHAR AS place_name,
a.relevance_score
FROM agg a
LEFT JOIN read_parquet('${lite_url}') l USING (pid)
WHERE a.m = ${plan.tokens.length}
`);
} else {
Expand Down Expand Up @@ -5873,11 +5873,11 @@ zoomWatcher = {
count(DISTINCT token) AS m
FROM contrib GROUP BY pid
)
SELECT a.pid, l.label, l.source,
CAST(l.place_name AS VARCHAR) AS place_name,
SELECT a.pid,
NULL::VARCHAR AS label, NULL::VARCHAR AS source,
NULL::VARCHAR AS place_name,
a.relevance_score
FROM agg a
LEFT JOIN read_parquet('${lite_url}') l USING (pid)
WHERE a.m = ${plan.tokens.length}
`);
}
Expand Down Expand Up @@ -6112,7 +6112,15 @@ zoomWatcher = {
// of PR #236 round 1). The string is captured here, before any
// `db.query` await, and re-used by both the SELECT and the
// COUNT below.
const sourceSQL = sourceFilterSQL('s.source');
// #171 perf fix: the substrate builder no longer joins lite at
// filter-build time (the 42k-row remote join made big-result
// queries SLOWER than baseline — benchmark 2026-07-11, Codex-
// verified). search_pids now carries NULL display columns for
// substrate searches, so every consumer below joins lite and
// filters on the COALESCEd value (interim rows still carry
// their own values; NULL-vs-value never changes interim
// semantics because COALESCE prefers s.*).
const sourceSQL = sourceFilterSQL("COALESCE(s.source, l.source)");
const facetSQL = facetFilterSQL();
// Telemetry-equivalent of the SQL snapshots above (`hadSourceFilter`
// / `hadFacetFilter`) is declared OUTSIDE this try block so it
Expand Down Expand Up @@ -6157,15 +6165,19 @@ zoomWatcher = {
results = await runWorldQuery();
} else {
results = await db.query(`
SELECT s.pid, s.label, s.source, l.latitude, l.longitude,
s.place_name, s.relevance_score
SELECT s.pid,
COALESCE(s.label, l.label) AS label,
COALESCE(s.source, l.source) AS source,
l.latitude, l.longitude,
COALESCE(s.place_name, CAST(l.place_name AS VARCHAR)) AS place_name,
s.relevance_score
FROM search_pids s
INNER JOIN read_parquet('${lite_url}') l USING (pid)
WHERE 1=1
${bboxSQL}
${sourceSQL}
${facetSQL}
ORDER BY s.relevance_score DESC, s.label
ORDER BY s.relevance_score DESC, COALESCE(s.label, l.label)
LIMIT 50
`);
effectiveQueryShape = 'area';
Expand All @@ -6177,21 +6189,50 @@ zoomWatcher = {

async function runWorldQuery() {
return db.query(`
${
// Codex review (lite-join deferral, round 2): the
// join-everything-then-filter shape is ONLY needed
// when substrate rows (NULL s.source) must satisfy an
// ACTIVE source filter that reads l.source. In every
// other case — the interim producer, or no source
// filter — keep the original cheap shape: filter +
// LIMIT 50 first, join lite for just those 50.
(window.__searchFilter?.substrate && sourceSQL !== '')
? `
SELECT s.pid,
COALESCE(s.label, l.label) AS label,
COALESCE(s.source, l.source) AS source,
l.latitude, l.longitude,
COALESCE(s.place_name, CAST(l.place_name AS VARCHAR)) AS place_name,
s.relevance_score
FROM search_pids s
LEFT JOIN read_parquet('${lite_url}') l USING (pid)
WHERE 1=1
${sourceSQL}
${facetSQL}
ORDER BY s.relevance_score DESC, COALESCE(s.label, l.label)
LIMIT 50`
: `
WITH matches AS (
SELECT s.pid, s.label, s.source, s.place_name,
s.relevance_score
FROM search_pids s
WHERE 1=1
${sourceSQL}
${facetSQL}
${window.__searchFilter?.substrate ? '' : sourceSQL.replaceAll('COALESCE(s.source, l.source)', 's.source')}
ORDER BY s.relevance_score DESC
LIMIT 50
)
SELECT m.pid, m.label, m.source, l.latitude, l.longitude,
m.place_name, m.relevance_score
SELECT m.pid,
COALESCE(m.label, l.label) AS label,
COALESCE(m.source, l.source) AS source,
l.latitude, l.longitude,
COALESCE(m.place_name, CAST(l.place_name AS VARCHAR)) AS place_name,
m.relevance_score
FROM matches m
LEFT JOIN read_parquet('${lite_url}') l USING (pid)
ORDER BY m.relevance_score DESC, m.label
ORDER BY m.relevance_score DESC, COALESCE(m.label, l.label)`
}
`);
}

Expand Down Expand Up @@ -6379,6 +6420,7 @@ zoomWatcher = {
: `
SELECT COUNT(*) AS n
FROM search_pids s
LEFT JOIN read_parquet('${lite_url}') l USING (pid)
WHERE 1=1
${sourceSQL}
${facetSQL}
Expand Down
Loading