diff --git a/package-lock.json b/package-lock.json index 4aed683..14d12fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9007,7 +9007,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.26.0", + "version": "0.27.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/README.md b/packages/plugin/README.md index 0e1d94b..66b3fef 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -136,6 +136,12 @@ rest: true # required for the @export-ed table REST endpoints enabled: true # record bot analytics at all: bot_request (ingress volume by host/bot/device), # bot_serve (outcome by source/cache-status/bot — origin offload + cache hit rate), and # page_age (ms since the served page rendered — freshness at serve, cache hits only) + + crawlStats: # crawl breadth: distinct URLs crawled per bot per UTC day (HyperLogLog, ~0.8% error) + enabled: true # also gated by analytics.enabled above; read via GET /prerender_admin/crawl-breadth?days=7 + flushInterval: 300000 # ms between sketch persists (max observation loss if a worker dies) + retentionDays: 90 # sketch rows older than this are swept at day rollover + maxBotsPerThread: 64 # cap on per-thread sketches; overflow bots share one '~overflow' bucket recordUnmatched: true # also record UAs that matched no configured bot (as 'other') bots: # registry: which crawlers are tracked by name. { name, match } — match is a - { name: Googlebot, match: googlebot } # case-insensitive UA substring; longer matches win. diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 4057351..2617ac0 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.26.0", + "version": "0.27.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/config.js b/packages/plugin/src/config.js index 040d6b0..faa9668 100644 --- a/packages/plugin/src/config.js +++ b/packages/plugin/src/config.js @@ -494,6 +494,19 @@ const defaultConfig = () => ({ { name: 'Sitebulb', match: 'sitebulb' }, ], }, + + // Crawl breadth: distinct URLs crawled per bot per UTC day, via per-thread HyperLogLog + // sketches flushed to crawl_stats.CrawlSketch (util/crawlStats.js). Read merged through + // GET /prerender_admin/crawl-breadth. Recording is additionally gated by the analytics + // gate above (no bot name → nothing to attribute a sketch to). + crawlStats: { + enabled: true, + flushInterval: 5 * MINUTE, // per-thread sketch persistence cadence (max data loss on a crash) + retentionDays: 90, // sketch rows older than this are swept at day rollover + // Sketches are 16 KB each; this caps a UA-derivation flood from minting unbounded + // per-thread sketches. Overflow bots share one '~overflow' bucket for the day. + maxBotsPerThread: 64, + }, }); // The live config object. Mutated in place by applyOptions so existing imports diff --git a/packages/plugin/src/http_handlers/bot_request.js b/packages/plugin/src/http_handlers/bot_request.js index a35e4d5..73ae338 100644 --- a/packages/plugin/src/http_handlers/bot_request.js +++ b/packages/plugin/src/http_handlers/bot_request.js @@ -13,6 +13,7 @@ import { fetchOriginResource } from '../util/upstream.js'; import { PrerenderedPage } from '../resources/PrerenderedPage.js'; import { resolveServingPolicy, pollForFreshRender } from '../util/renderNow.js'; import { currentMinuteMs } from '../util/time.js'; +import { recordCrawl } from '../util/crawlStats.js'; import { deliverResource } from './response.js'; export async function handleBotRequest(request) { @@ -29,6 +30,9 @@ export async function handleBotRequest(request) { const recordBots = config.analytics.enabled && (request.botName !== 'other' || config.analytics.recordUnmatched); if (recordBots) { server.recordAnalytics(true, 'bot_request', url.hostname, request.botName, deviceType); + // Crawl breadth (distinct URLs per bot per day): one hash + one byte max into a + // per-thread HLL sketch — see util/crawlStats.js for the cost/loss model. + recordCrawl(request.botName, cacheUrl); } // Debug/observability info surfaced as x-harper-* response headers (only when the diff --git a/packages/plugin/src/resources/PrerenderAdmin.js b/packages/plugin/src/resources/PrerenderAdmin.js index c7648a8..9efddc5 100644 --- a/packages/plugin/src/resources/PrerenderAdmin.js +++ b/packages/plugin/src/resources/PrerenderAdmin.js @@ -24,6 +24,7 @@ * GET /prerender_admin/pages ?prefix&cursor&limit super_user * GET /prerender_admin/page-content ?cacheKey (text/plain) super_user * GET /prerender_admin/unrouted this worker's unrouted tally super_user + * GET /prerender_admin/crawl-breadth ?days (default 7, max 31) super_user * POST /prerender_admin/explain { url, deviceType } super_user * POST /prerender_admin/schedule { cacheKey } -> local row super_user * POST /prerender_admin/queue { scope, paused } super_user @@ -64,6 +65,7 @@ import { fetchScheduleFromPeer } from '../util/peer.js'; import { getLastReconcile, isReconcileRunning, runReconcileOnce } from '../util/reconcile.js'; import { getBacklogSnapshotState, runBacklogSnapshotOnce } from '../util/backlogSnapshot.js'; import { peekUnroutedReport } from '../util/unrouted.js'; +import { mergeBreadthRow, finalizeBreadth } from '../util/crawlStats.js'; import { decode } from '../util/contentEncoding.js'; import { RenderQueue } from './RenderQueue.js'; import { QueueState } from './QueueState.js'; @@ -358,6 +360,8 @@ export class PrerenderAdmin extends Resource { interval: config.ingress.report.interval, report: peekUnroutedReport(), }); + case 'crawl-breadth': + return PrerenderAdmin.crawlBreadth(target); default: return json({ error: `Unknown route: ${route}` }, 404); } @@ -1041,6 +1045,51 @@ export class PrerenderAdmin extends Resource { * would be unindexed table filters, and the client filters the fetched page instead, * labelled as exactly that. `content` is never selected. */ + // Crawl breadth: distinct URLs crawled per bot per UTC day, from the merged CrawlSketch + // node rows (util/crawlStats.js). Query cost: one day-indexed range read of + // days × bots-with-traffic × nodes 16 KB rows (a week on a 4-node cluster is a few + // hundred rows), capped below and reported truncated rather than presented as complete. + // Never touches the render queue or the page cache. + static crawlBreadth(target) { + return withHeavySlot(() => this.crawlBreadthInner(target)); + } + + static async crawlBreadthInner(target) { + const days = Math.min(Math.max(1, Number(target?.get?.('days')) || 7), 31); + const since = new Date(Date.now() - (days - 1) * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + const cap = 4096; + const results = databases.crawl_stats.CrawlSketch.search({ + conditions: [{ attribute: 'day', comparator: 'greater_than_equal', value: since }], + select: ['day', 'bot', 'registers'], + limit: cap + 1, // one extra row = "truncated", never merged + }); + + // Stream the cursor instead of buffering it: each 16 KB row merges into the + // accumulator and is released, so the resident set is one sketch per (day, bot) — + // not up to cap × 16 KB of raw rows. Yield the event loop periodically; this worker + // also serves bot traffic. + const byDay = new Map(); + let shardsMerged = 0; + let truncated = false; + for await (const row of results) { + if (shardsMerged === cap) { + truncated = true; + break; + } + mergeBreadthRow(byDay, row); + if (++shardsMerged % 200 === 0) await yieldNow(); + } + + return json({ + node: server.hostname, + days, + since, + shardsMerged, + truncated, + breadth: finalizeBreadth(byDay), + }); + } + static listPages(target) { return withHeavySlot(() => this.listPagesInner(target)); } diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index 91658a3..8f5231c 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -147,3 +147,23 @@ type PrerenderedPage @table(database: "page_cache") @export { type SharedBuffer @table(database: "coordination", replicate: false) { key: Any @primaryKey } + +# One HyperLogLog register array per (UTC day, bot, node) — this node's shard of that day's +# global distinct-URL sketch for that crawler (util/crawlStats.js). Workers merge their +# thread sketches into the node row under the cross-worker mutex; readers merge a day's +# node rows by element-wise max, which reassembles the exact union. Rows replicate +# (default) so any one node can answer for the cluster. Deliberately NOT @export: the raw +# 16 KB shards are only useful merged — reads go through GET /prerender_admin/crawl-breadth, +# behind that resource's super-user gate. Own database: writes are low-frequency and share +# no transaction with any other table. +type CrawlSketch @table(database: "crawl_stats") { + id: String @primaryKey # `${day}|${bot}|${node}` + day: String @indexed + bot: String + node: String + registers: Bytes + # This NODE's own estimate, stored for eyeballing a raw row. Never sum these — + # distinct counts don't add; merge the node rows and estimate the union. + estimate: Float + updatedAt: Date +} diff --git a/packages/plugin/src/util/crawlStats.js b/packages/plugin/src/util/crawlStats.js new file mode 100644 index 0000000..9cb3386 --- /dev/null +++ b/packages/plugin/src/util/crawlStats.js @@ -0,0 +1,211 @@ +/** + * Crawl breadth: distinct URLs crawled per bot per UTC day, via per-thread HyperLogLog + * sketches (util/hll.js — see there for why a sketch and why per-thread shards merge into + * an exact global union). + * + * Shape: each worker thread keeps one in-memory sketch per bot for the current day and, on + * a timer, merges it into this node's `crawl_stats.CrawlSketch` row (`day|bot|node`) — + * read-merge-write under the cross-worker mutex (util/coordination.js), so concurrent + * workers never lose each other's registers. One row per node (not per thread) keeps the + * read side inside the admin console's query-cost rules: a week is days × bots × nodes + * small rows, not × threads. Cross-node there is no contention at all — the node is in the + * key. The read side (PrerenderAdmin's crawl-breadth route) merges a day's rows per bot. + * + * Hot-path cost (recordCrawl): one number compare for day rollover, one Map lookup, one + * 53-bit string hash + one byte max. No allocation, no await, no storage touch. Everything + * async (flush, retention sweep) happens on the timer or via setImmediate. + * + * Loss model: a thread dying loses at most flushInterval worth of that thread's unmerged + * observations — an undercount, never a double count. Thread sketches are cumulative for + * the day and merging is idempotent, so a failed flush is simply retried whole next cycle. + * Rows replicate (table default), so any single node can answer for the cluster. + */ + +import { config } from '../config.js'; +import { getMutex } from './coordination.js'; +import { createSketch, addToSketch, mergeSketch, estimateSketch } from './hll.js'; + +// When a thread sees more distinct bot names than the cap in one day (a UA-derivation +// flood — registry bots can't exceed it), the overflow shares this bucket. Distinct from +// the registry's 'other' so a capped day is visible as such rather than blending in. +export const OVERFLOW_BUCKET = '~overflow'; + +let sketches = new Map(); // botName -> Uint8Array registers +const dirty = new Set(); // botNames with unflushed observations +let day = null; // 'YYYY-MM-DD' (UTC) the current sketches belong to +let dayEndMs = 0; // rollover boundary, so the hot path compares one number +let flushTimer = null; + +const utcDayOf = (ms) => new Date(ms).toISOString().slice(0, 10); +const table = () => databases.crawl_stats.CrawlSketch; + +/** + * Observe one crawled URL. Called on the bot serving path (behind the analytics gate) — + * everything above about hot-path cost is about this function. Synchronous by design. + */ +export function recordCrawl(botName, url) { + if (!config.crawlStats.enabled) return; + + const now = Date.now(); + if (now >= dayEndMs) rollover(now); + + let sketch = sketches.get(botName); + if (!sketch) { + if (sketches.size >= config.crawlStats.maxBotsPerThread) { + botName = OVERFLOW_BUCKET; + sketch = sketches.get(botName); + } + if (!sketch) { + sketch = createSketch(); + sketches.set(botName, sketch); + } + } + addToSketch(sketch, url); + dirty.add(botName); + + // Lazily started so only threads that actually serve bot traffic run a timer. + if (!flushTimer) { + flushTimer = setInterval(() => flushSketches().catch((e) => logger.error(e)), config.crawlStats.flushInterval); + flushTimer.unref?.(); + } +} + +// Close out the old day and start the new one. The old sketches are captured and persisted +// off the hot path via setImmediate; the maps are swapped synchronously so no observation +// lands in the wrong day. Also the retention hook: one worker per node sweeps expired rows. +function rollover(now) { + const previous = day ? { day, sketches, dirty: [...dirty] } : null; + day = utcDayOf(now); + dayEndMs = Date.parse(day) + 24 * 60 * 60 * 1000; + sketches = new Map(); + dirty.clear(); + + if (previous?.dirty.length) { + setImmediate(() => persist(previous.day, previous.sketches, previous.dirty).catch((e) => logger.error(e))); + } + if (previous && server.workerIndex === 0) { + setImmediate(() => sweepExpired().catch((e) => logger.error(e))); + } +} + +/** Persist this thread's dirty sketches for the current day. Exported for tests. */ +export async function flushSketches() { + if (!dirty.size) return; + const bots = [...dirty]; + dirty.clear(); + try { + await persist(day, sketches, bots); + } catch (e) { + // Sketches are cumulative for the day and merging is idempotent, so retrying the + // whole flush next cycle is safe — re-mark rather than wait for new traffic. + for (const bot of bots) dirty.add(bot); + throw e; + } +} + +async function persist(forDay, forSketches, bots) { + const CrawlSketch = table(); + for (const bot of bots) { + const mine = forSketches.get(bot); + if (!mine) continue; + // Read-merge-write of this node's row, serialized against the other workers by the + // cross-worker mutex. The row is this node's own (node is in the key), so the read + // is local — no cross-node fetch to time out on. Explicit lock/finally rather than + // `withLock` because this is a one-shot section — the repo's withLock idiom wraps a + // REUSABLE function (see RenderQueue.claim), and an inline `withLock(fn)()` reads + // like an accidental double call. + const id = `${forDay}|${bot}|${server.hostname}`; + const mutex = getMutex(`crawlSketch/${bot}`); + await mutex.lock(); + try { + const existing = await CrawlSketch.get(id); + // Merge into a copy: merging the row INTO the thread sketch would fold other + // workers' registers into thread-local state, and a later failed write would + // then re-contribute them as if they were this thread's own. + const merged = new Uint8Array(mine); + if (existing?.registers) mergeSketch(merged, existing.registers); + // `estimate` is this NODE's count, stored for eyeballing a raw row — the real + // per-day number requires merging all nodes first (distinct counts don't add). + await CrawlSketch.put(id, { + day: forDay, + bot, + node: server.hostname, + registers: merged, + estimate: estimateSketch(merged), + updatedAt: Date.now(), + }); + } finally { + mutex.unlock(); + } + } +} + +// Delete rows past retention. Runs once per day-rollover on one worker per node; deletes +// are idempotent so concurrent nodes sweeping the same replicated rows is harmless. Bounded +// per pass — anything left over is caught the next day. +async function sweepExpired() { + const CrawlSketch = table(); + const cutoff = utcDayOf(Date.now() - config.crawlStats.retentionDays * 24 * 60 * 60 * 1000); + const expired = await CrawlSketch.search({ + conditions: [{ attribute: 'day', comparator: 'less_than', value: cutoff }], + select: ['id'], + limit: 1000, + }); + for await (const row of expired) { + await CrawlSketch.delete(row.id); + } +} + +/** + * Streaming accumulator for the admin crawl-breadth route: fold one raw sketch row into + * `byDay` (a Map of day -> Map of bot -> { registers, shards }). Split this way so the + * route can `for await` a search cursor row by row — merging each 16 KB row into the + * accumulator and releasing it — instead of buffering the whole result set; the resident + * set is one sketch per (day, bot), not one per row. + */ +export function mergeBreadthRow(byDay, row) { + if (!row?.registers || !row.bot) return; + let bots = byDay.get(row.day); + if (!bots) byDay.set(row.day, (bots = new Map())); + let entry = bots.get(row.bot); + if (!entry) bots.set(row.bot, (entry = { registers: createSketch(), shards: 0 })); + mergeSketch(entry.registers, row.registers); + entry.shards++; +} + +/** + * Turn the accumulator into per-day breadth estimates: + * [{ day, total, bots: [{ bot, distinctUrls, shards }] }] sorted by day desc, where + * `total` is the distinct-URL count of the UNION across bots (not a sum). + */ +export function finalizeBreadth(byDay) { + return [...byDay] + .sort(([a], [b]) => (a < b ? 1 : -1)) + .map(([forDay, bots]) => { + const union = createSketch(); + const perBot = [...bots] + .map(([bot, { registers, shards }]) => { + mergeSketch(union, registers); + return { bot, distinctUrls: estimateSketch(registers), shards }; + }) + .sort((a, b) => b.distinctUrls - a.distinctUrls); + return { day: forDay, total: estimateSketch(union), bots: perBot }; + }); +} + +/** Convenience over the two halves above, for in-memory row sets (and tests). */ +export function computeBreadth(rows) { + const byDay = new Map(); + for (const row of rows) mergeBreadthRow(byDay, row); + return finalizeBreadth(byDay); +} + +/** Test hook: reset module state between tests. */ +export function resetCrawlStats() { + sketches = new Map(); + dirty.clear(); + day = null; + dayEndMs = 0; + if (flushTimer) clearInterval(flushTimer); + flushTimer = null; +} diff --git a/packages/plugin/src/util/hll.js b/packages/plugin/src/util/hll.js new file mode 100644 index 0000000..0eedf5c --- /dev/null +++ b/packages/plugin/src/util/hll.js @@ -0,0 +1,95 @@ +/** + * HyperLogLog distinct-count sketch — the mechanism behind the crawl-breadth metric + * (distinct URLs crawled per bot per day, see util/crawlStats.js). + * + * Why a sketch and not a counter or a table: distinct counts don't add (summing per-thread + * tallies double-counts every URL two threads both saw), a URL-dimensioned analytics key + * space would be the whole corpus (~10^6 keys per flush window), and a row-per-URL table + * would put a storage write on the bot read path. An HLL register array is a fixed 16 KB + * per (bot, day), costs one string hash + one byte max per observation, and — the property + * everything here leans on — merges LOSSLESSLY by element-wise max: merging two sketches + * yields byte-for-byte the sketch a single observer of both streams would have built, so + * per-thread/per-node shards reassemble into one exact-union global sketch at read time. + * Merging never compounds the estimation error. + * + * Parameters: p = 14 → m = 16384 registers, standard error ≈ 1.04/√m ≈ 0.8%. The hash is + * cyrb53 (53-bit, imul-based — no BigInt, no allocation): 14 bits pick the register, the + * remaining 39 bits feed the rank, so register saturation is not reachable at any realistic + * URL-corpus cardinality (rank caps at 40 ≈ 2^39·m distinct values). The classic 32-bit + * large-range correction is deliberately omitted: it exists for hash spaces the estimate + * can approach, and 2^53 is not one. + * + * Estimation is the original Flajolet et al. formula with the small-range fallback + * (linear counting while empty registers remain), which is where a day's sketch for a + * low-traffic bot actually lives — so that path is tested as carefully as the asymptotic one. + */ + +export const HLL_P = 14; +export const HLL_REGISTERS = 1 << HLL_P; // 16384 +const INDEX_MASK = HLL_REGISTERS - 1; +const RANK_BITS = 53 - HLL_P; // 39 +const ALPHA = 0.7213 / (1 + 1.079 / HLL_REGISTERS); + +// 2^-r for r in [0, 40] — estimate() is read-path only, but there's no reason to pay +// Math.pow in a 16k-iteration loop either. +const POW2_NEG = new Float64Array(RANK_BITS + 2); +for (let r = 0; r < POW2_NEG.length; r++) POW2_NEG[r] = 2 ** -r; + +/** 53-bit string hash (cyrb53, public domain — bryc). Deterministic, allocation-free. */ +export function hash53(str, seed = 0) { + let h1 = 0xdeadbeef ^ seed; + let h2 = 0x41c6ce57 ^ seed; + for (let i = 0; i < str.length; i++) { + const ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} + +export const createSketch = () => new Uint8Array(HLL_REGISTERS); + +/** Observe one value. The entire hot-path cost of the crawl-breadth metric lives here. */ +export function addToSketch(registers, value) { + const h = hash53(value); + const index = h & INDEX_MASK; + // The remaining 39 bits, exact: h < 2^53 so this division is integer-safe. + const rest = Math.floor(h / HLL_REGISTERS); + // rank = leading zeros within the 39-bit field + 1; rest === 0 → all zeros → RANK_BITS + 1. + const hi = Math.floor(rest / 4294967296); // top 7 bits + const bitLength = hi !== 0 ? 64 - Math.clz32(hi) : rest !== 0 ? 32 - Math.clz32(rest) : 0; + const rank = RANK_BITS - bitLength + 1; + if (rank > registers[index]) registers[index] = rank; +} + +/** + * Union `src` into `dest` (element-wise max), in place. Merging shards is exact with + * respect to set union — see the module comment. + */ +export function mergeSketch(dest, src) { + for (let i = 0; i < HLL_REGISTERS; i++) { + if (src[i] > dest[i]) dest[i] = src[i]; + } + return dest; +} + +/** Estimated cardinality of the set the sketch observed. */ +export function estimateSketch(registers) { + let sum = 0; + let zeros = 0; + for (let i = 0; i < HLL_REGISTERS; i++) { + const r = registers[i]; + sum += POW2_NEG[r]; + if (r === 0) zeros++; + } + const raw = (ALPHA * HLL_REGISTERS * HLL_REGISTERS) / sum; + // Small-range: linear counting is the better estimator while empty registers remain. + if (raw <= 2.5 * HLL_REGISTERS && zeros > 0) { + return Math.round(HLL_REGISTERS * Math.log(HLL_REGISTERS / zeros)); + } + return Math.round(raw); +} diff --git a/packages/plugin/test/crawlStats.test.js b/packages/plugin/test/crawlStats.test.js new file mode 100644 index 0000000..9df5de1 --- /dev/null +++ b/packages/plugin/test/crawlStats.test.js @@ -0,0 +1,189 @@ +import { test, before, beforeEach, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import { setImmediate as tick } from 'node:timers/promises'; + +/** + * crawlStats — the per-thread sketch state machine and its persistence contract. + * + * The properties pinned here: + * - flush merges into the node row via read-merge-write (a second worker's flush must + * UNION with what's stored, never overwrite it — that is what makes the node row the + * union of all workers); + * - the flush is serialized by the cross-worker mutex; + * - the per-thread bot cap folds overflow into '~overflow' instead of minting sketches; + * - UTC day rollover persists the old day's sketches and starts clean ones; + * - a failed flush re-marks its bots dirty so the next cycle retries the whole + * (cumulative, idempotent) sketch; + * - computeBreadth groups by day, merges shards per bot, and reports the cross-bot + * union as `total` (never a sum). + */ + +const rows = new Map(); +let locks = []; + +let recordCrawl, flushSketches, computeBreadth, resetCrawlStats, OVERFLOW_BUCKET; +let estimateSketch, createSketch, addToSketch; +let applyOptions; + +before(async () => { + globalThis.Resource = class {}; + globalThis.server = { hostname: 'node-a', workerIndex: 1 }; + globalThis.logger = { info() {}, warn() {}, error() {} }; + globalThis.databases = { + coordination: { + SharedBuffer: { + primaryStore: { + tryLock: (key) => { + locks.push(key); + return true; // granted synchronously; the callback is never called + }, + unlock() {}, + }, + }, + }, + crawl_stats: { + CrawlSketch: { + async get(id) { + const row = rows.get(id); + return row ? { ...row } : null; + }, + async put(id, data) { + rows.set(id, { ...data }); + }, + async delete(id) { + rows.delete(id); + }, + async search() { + return []; + }, + }, + }, + }; + ({ applyOptions } = await import('../src/config.js')); + ({ recordCrawl, flushSketches, computeBreadth, resetCrawlStats, OVERFLOW_BUCKET } = await import( + '../src/util/crawlStats.js' + )); + ({ estimateSketch, createSketch, addToSketch } = await import('../src/util/hll.js')); +}); + +beforeEach(() => { + applyOptions({}); + resetCrawlStats(); + rows.clear(); + locks = []; + mock.timers.reset(); +}); + +const today = () => new Date().toISOString().slice(0, 10); + +test('flush writes the node row with an accurate estimate, under the mutex', async () => { + for (let i = 0; i < 1000; i++) recordCrawl('Googlebot', `https://site.example.com/p/${i}`); + // Duplicates must not move it. + for (let i = 0; i < 1000; i++) recordCrawl('Googlebot', `https://site.example.com/p/${i}`); + await flushSketches(); + + const row = rows.get(`${today()}|Googlebot|node-a`); + assert.ok(row, 'node row written'); + assert.equal(row.bot, 'Googlebot'); + assert.equal(row.node, 'node-a'); + assert.ok(Math.abs(row.estimate - 1000) / 1000 <= 0.03, `estimate ${row.estimate}`); + assert.ok( + locks.some((k) => k.includes('crawlSketch/Googlebot')), + 'flush took the cross-worker mutex' + ); +}); + +test("flush UNIONS with the stored row — another worker's registers survive", async () => { + // Simulate worker 1: URLs [0, 500). + for (let i = 0; i < 500; i++) recordCrawl('Googlebot', `https://site.example.com/p/${i}`); + await flushSketches(); + // Simulate worker 2 (fresh thread state): overlapping URLs [250, 750). + resetCrawlStats(); + for (let i = 250; i < 750; i++) recordCrawl('Googlebot', `https://site.example.com/p/${i}`); + await flushSketches(); + + const row = rows.get(`${today()}|Googlebot|node-a`); + // Union is 750 distinct — an overwrite would read ~500, a sum-like error ~1000. + assert.ok(Math.abs(row.estimate - 750) / 750 <= 0.03, `estimate ${row.estimate}`); +}); + +test('bot cap folds overflow into the overflow bucket', async () => { + applyOptions({ crawlStats: { maxBotsPerThread: 2 } }); + recordCrawl('Googlebot', 'https://site.example.com/a'); + recordCrawl('Bingbot', 'https://site.example.com/b'); + recordCrawl('SomeDerivedBot', 'https://site.example.com/c'); + recordCrawl('AnotherDerivedBot', 'https://site.example.com/d'); + await flushSketches(); + + assert.ok(rows.has(`${today()}|Googlebot|node-a`)); + assert.ok(rows.has(`${today()}|Bingbot|node-a`)); + assert.ok(!rows.has(`${today()}|SomeDerivedBot|node-a`), 'overflow bot must not mint a sketch'); + const overflow = rows.get(`${today()}|${OVERFLOW_BUCKET}|node-a`); + assert.ok(overflow, 'overflow bucket written'); + assert.equal(overflow.estimate, 2); +}); + +test('disabled config records nothing', async () => { + applyOptions({ crawlStats: { enabled: false } }); + recordCrawl('Googlebot', 'https://site.example.com/a'); + await flushSketches(); + assert.equal(rows.size, 0); +}); + +test('UTC day rollover persists the old day and starts clean', async (t) => { + t.mock.timers.enable({ apis: ['Date'], now: Date.parse('2026-08-04T23:59:00Z') }); + recordCrawl('Googlebot', 'https://site.example.com/old-day'); + + t.mock.timers.setTime(Date.parse('2026-08-05T00:00:01Z')); + recordCrawl('Googlebot', 'https://site.example.com/new-day'); + await tick(); // rollover persists the previous day via setImmediate + await flushSketches(); + + const oldRow = rows.get('2026-08-04|Googlebot|node-a'); + const newRow = rows.get('2026-08-05|Googlebot|node-a'); + assert.ok(oldRow, 'old day persisted at rollover'); + assert.ok(newRow, 'new day written by the next flush'); + assert.equal(oldRow.estimate, 1); + assert.equal(newRow.estimate, 1, 'new day must not inherit the old sketch'); +}); + +test('a failed flush re-marks its bots dirty and the retry succeeds', async () => { + recordCrawl('Googlebot', 'https://site.example.com/a'); + const put = databases.crawl_stats.CrawlSketch.put; + databases.crawl_stats.CrawlSketch.put = async () => { + throw new Error('storage hiccup'); + }; + await assert.rejects(flushSketches()); + databases.crawl_stats.CrawlSketch.put = put; + await flushSketches(); // must retry without new traffic + assert.ok(rows.has(`${today()}|Googlebot|node-a`)); +}); + +test('computeBreadth merges shards per bot and reports the cross-bot union', () => { + const urls = (from, to) => { + const s = createSketch(); + for (let i = from; i < to; i++) addToSketch(s, `https://site.example.com/p/${i}`); + return s; + }; + const breadth = computeBreadth([ + // Googlebot, two node shards with overlap: union 1500 distinct. + { day: '2026-08-04', bot: 'Googlebot', registers: urls(0, 1000) }, + { day: '2026-08-04', bot: 'Googlebot', registers: urls(500, 1500) }, + // GPTBot crawled a subset Googlebot also crawled: cross-bot union stays 1500. + { day: '2026-08-04', bot: 'GPTBot', registers: urls(0, 300) }, + { day: '2026-08-03', bot: 'Googlebot', registers: urls(0, 100) }, + ]); + + assert.equal(breadth.length, 2); + assert.equal(breadth[0].day, '2026-08-04', 'sorted newest first'); + const [google, gpt] = breadth[0].bots; + assert.equal(google.bot, 'Googlebot'); + assert.equal(google.shards, 2); + assert.ok(Math.abs(google.distinctUrls - 1500) / 1500 <= 0.03); + assert.equal(gpt.bot, 'GPTBot'); + // Union across bots, not a sum: 1500 + 300 overlapping = 1500. + assert.ok(Math.abs(breadth[0].total - 1500) / 1500 <= 0.03, `total ${breadth[0].total}`); + assert.equal(breadth[1].bots[0].distinctUrls, 100); + // estimateSketch sanity on the exported surface used by the admin route. + assert.equal(estimateSketch(createSketch()), 0); +}); diff --git a/packages/plugin/test/hll.test.js b/packages/plugin/test/hll.test.js new file mode 100644 index 0000000..f1b61f3 --- /dev/null +++ b/packages/plugin/test/hll.test.js @@ -0,0 +1,76 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { HLL_REGISTERS, createSketch, addToSketch, mergeSketch, estimateSketch, hash53 } from '../src/util/hll.js'; + +/** + * The properties the crawl-breadth metric leans on: + * - estimates land within tolerance at small (linear-counting), mid, and large + * cardinalities — standard error at p=14 is ~0.8%, asserted here at 3% to keep the + * test deterministic-in-practice without being flaky; + * - duplicates never move the estimate (the whole point vs a counter); + * - merge is EXACTLY the sketch of the union, byte for byte — the property that makes + * per-thread/per-node shards a lossless decomposition of one global sketch. + */ + +const url = (i) => `https://site.example.com/product/${i}?variant=${i % 7}`; + +const sketchOf = (from, to) => { + const s = createSketch(); + for (let i = from; i < to; i++) addToSketch(s, url(i)); + return s; +}; + +const assertWithin = (estimate, actual, tolerance) => { + const error = Math.abs(estimate - actual) / actual; + assert.ok(error <= tolerance, `estimate ${estimate} vs actual ${actual}: error ${(error * 100).toFixed(2)}%`); +}; + +test('hash53 is deterministic and spreads', () => { + assert.equal(hash53('https://site.example.com/a'), hash53('https://site.example.com/a')); + assert.notEqual(hash53('https://site.example.com/a'), hash53('https://site.example.com/b')); + assert.ok(Number.isSafeInteger(hash53('x'.repeat(2000)))); +}); + +test('empty sketch estimates zero', () => { + assert.equal(estimateSketch(createSketch()), 0); +}); + +test('small cardinality (linear-counting range) is accurate', () => { + // A day's sketch for a low-traffic bot lives here, so this range matters most. + assertWithin(estimateSketch(sketchOf(0, 100)), 100, 0.03); + assertWithin(estimateSketch(sketchOf(0, 5000)), 5000, 0.03); +}); + +test('mid and large cardinality are accurate', () => { + assertWithin(estimateSketch(sketchOf(0, 100_000)), 100_000, 0.03); + // The full-corpus scale (~10^6 URLs). + assertWithin(estimateSketch(sketchOf(0, 1_000_000)), 1_000_000, 0.03); +}); + +test('duplicates never move the estimate', () => { + const once = sketchOf(0, 10_000); + const thrice = createSketch(); + for (let pass = 0; pass < 3; pass++) { + for (let i = 0; i < 10_000; i++) addToSketch(thrice, url(i)); + } + assert.deepEqual(thrice, once); +}); + +test('merge is byte-for-byte the sketch of the union', () => { + // Overlapping shards: [0, 60k) and [40k, 100k) — 20k shared URLs must collapse. + const a = sketchOf(0, 60_000); + const b = sketchOf(40_000, 100_000); + const merged = mergeSketch(new Uint8Array(a), b); + assert.deepEqual(merged, sketchOf(0, 100_000)); + assertWithin(estimateSketch(merged), 100_000, 0.03); +}); + +test('merge order and grouping are irrelevant', () => { + const shards = [sketchOf(0, 10_000), sketchOf(5000, 20_000), sketchOf(15_000, 30_000)]; + const forward = createSketch(); + for (const s of shards) mergeSketch(forward, s); + const backward = createSketch(); + for (const s of [...shards].reverse()) mergeSketch(backward, s); + assert.deepEqual(forward, backward); + assert.equal(forward.length, HLL_REGISTERS); +});