From 73f86c7af1d3ab01b40e357e236149473be9db71 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:22:17 +0100 Subject: [PATCH 01/28] fix(webapp): stop saving global flags from unsetting the locked ones (#4751) ## Summary On a self-hosted instance, saving anything on the global admin feature flags page also deleted the two read-only flags, `defaultWorkerInstanceGroupId` and `taskEventRepository`. Losing the first one leaves deployed runs with no default worker group. Neither deletion showed up in the confirm dialog, so the flags disappeared silently. ## Root cause The page submits only the flags its UI is managing, and strips the read-only ones from the payload unless "Unlock read-only flags" is ticked. The action treated every catalog key absent from that payload as "the admin unset this", and protected the locked keys only when the instance was managed cloud. Anywhere else, both locked rows fell straight into the delete sweep. The protection now keys off what the client says it was editing rather than off the deployment: ```ts const canDeleteLocked = params.unlockLockedFlags && !params.isManagedCloud; ... } else if (canDeleteLocked || !GLOBAL_LOCKED_FLAGS.includes(key)) { keysToDelete.push(key); } ``` Exactly one case changes: a locked flag, on a non managed-cloud instance, with the flags not unlocked, is now kept instead of deleted. Managed cloud behaviour is bit for bit identical, and ticking the unlock box still gives a self-hosted instance full control. The write moves into `replaceGlobalFeatureFlags` so it can be driven directly in tests against a real Postgres. --- .../webapp/app/routes/admin.feature-flags.tsx | 64 ++++---- apps/webapp/app/v3/featureFlags.server.ts | 47 +++++- .../test/adminFeatureFlagsRouteAction.test.ts | 137 ++++++++++++++++ .../globalFeatureFlagsLockedFlags.test.ts | 150 ++++++++++++++++++ 4 files changed, 360 insertions(+), 38 deletions(-) create mode 100644 apps/webapp/test/adminFeatureFlagsRouteAction.test.ts create mode 100644 apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index ef8caec4bd4..197800c7ef3 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -5,17 +5,18 @@ import { json } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { LockClosedIcon } from "@heroicons/react/20/solid"; -import { boundedIn, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; import { FEATURE_FLAG, GLOBAL_LOCKED_FLAGS, + type FeatureFlagKey, type FlagControlType, getAllFlagControlTypes, validatePartialFeatureFlags, } from "~/v3/featureFlags"; -import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; +import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; import { featuresForRequest } from "~/features.server"; import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; @@ -38,6 +39,12 @@ import { type WorkerGroup, } from "~/components/admin/FlagControls"; +/** What the page posts to the action. See the note on payloadSchema. */ +type SaveFlagsBody = { + flags: Record; + unlockLockedFlags: boolean; +}; + export const loader = dashboardLoader( { authorization: { requireSuper: true } }, async ({ request }) => { @@ -87,7 +94,16 @@ export const action = dashboardAction( return json({ error: "Invalid JSON body" }, { status: 400 }); } - const payloadSchema = z.object({ flags: z.record(z.unknown()) }); + // The zod schema leaves unlockLockedFlags optional so a tab opened before this shipped still + // saves, defaulting to the safe answer. SaveFlagsBody keeps it required for our own client, so + // dropping it from the page is a compile error rather than a silently disabled unlock. + const payloadSchema = z.object({ + flags: z.record(z.unknown()), + // The page only submits the flags it is managing, so an omitted key is ambiguous for the + // locked flags: this says whether the admin unlocked them and is therefore authoritative + // over them too. + unlockLockedFlags: z.boolean().optional(), + }); const parsed = payloadSchema.safeParse(body); if (!parsed.success) { return json({ error: "Invalid payload" }, { status: 400 }); @@ -116,39 +132,12 @@ export const action = dashboardAction( ); } - const validatedFlags = validationResult.data as Record; - const controlTypes = getAllFlagControlTypes(); - const catalogKeys = Object.keys(controlTypes); - - const keysToDelete: string[] = []; - const upsertOps: ReturnType[] = []; - - for (const key of catalogKeys) { - if (key in validatedFlags) { - upsertOps.push( - prisma.featureFlag.upsert({ - where: { key }, - create: { key, value: validatedFlags[key] as any }, - update: { value: validatedFlags[key] as any }, - }) - ); - } else { - // On cloud, never delete locked flags (they're not in the payload - // because the UI doesn't include them). Locally, delete everything - // the user didn't include - full control. - const isProtected = isManagedCloud && GLOBAL_LOCKED_FLAGS.includes(key); - if (!isProtected) { - keysToDelete.push(key); - } - } - } - - await prisma.$transaction([ - ...upsertOps, - ...(keysToDelete.length > 0 - ? [prisma.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })] - : []), - ]); + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: validationResult.data as Record, + catalogKeys: Object.keys(getAllFlagControlTypes()) as FeatureFlagKey[], + isManagedCloud, + unlockLockedFlags: parsed.data.unlockLockedFlags ?? false, + }); return json({ success: true }); } @@ -213,7 +202,8 @@ export default function AdminFeatureFlagsRoute() { }; const handleSave = () => { - saveFetcher.submit(JSON.stringify({ flags: values }), { + const body: SaveFlagsBody = { flags: values, unlockLockedFlags: unlocked }; + saveFetcher.submit(JSON.stringify(body), { method: "POST", encType: "application/json", }); diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index fdd302ede84..dd1fb125ba6 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -1,11 +1,12 @@ import { type z } from "zod"; import type { PrismaClient } from "@trigger.dev/database"; -import { prisma, type PrismaClientOrTransaction } from "~/db.server"; +import { boundedIn, prisma, type PrismaClientOrTransaction } from "~/db.server"; import { FEATURE_FLAG, type FeatureFlagCatalogSchema, type FeatureFlagKey, FeatureFlagCatalog, + GLOBAL_LOCKED_FLAGS, validatePartialFeatureFlags, } from "~/v3/featureFlags"; import { env } from "~/env.server"; @@ -223,6 +224,50 @@ export async function applyGlobalMintKindFlip( }); } +/** + * Replace-semantics write for the global admin flags page: catalog keys present in + * `requestedFlags` are upserted, catalog keys absent from it are deleted. + * + * A locked flag absent from the payload means the page never offered it for editing, not that + * the admin unset it, so it survives the sweep. Only a self-hosted page that says it unlocked + * them can delete one. + */ +export async function replaceGlobalFeatureFlags( + client: PrismaClient, + params: { + requestedFlags: Record; + catalogKeys: FeatureFlagKey[]; + isManagedCloud: boolean; + unlockLockedFlags: boolean; + } +): Promise { + const canDeleteLocked = params.unlockLockedFlags && !params.isManagedCloud; + const upsertOps: ReturnType[] = []; + const keysToDelete: string[] = []; + + for (const key of params.catalogKeys) { + if (key in params.requestedFlags) { + const value = params.requestedFlags[key]; + upsertOps.push( + client.featureFlag.upsert({ + where: { key }, + create: { key, value: value as any }, + update: { value: value as any }, + }) + ); + } else if (canDeleteLocked || !GLOBAL_LOCKED_FLAGS.includes(key)) { + keysToDelete.push(key); + } + } + + await client.$transaction([ + ...upsertOps, + ...(keysToDelete.length > 0 + ? [client.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })] + : []), + ]); +} + /** The global flag set, with the env-var defaults this app applies. */ export async function globalFeatureFlags() { return flags({ diff --git a/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts new file mode 100644 index 00000000000..a510fbe0f35 --- /dev/null +++ b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts @@ -0,0 +1,137 @@ +// The page posts only the flags its UI manages, so how the action reads an absent key is the whole +// bug surface. These drive the real exported action against a real Postgres and assert on the rows +// it leaves behind. The only module substituted is the auth wrapper, so the handler can be called +// without a super-admin session; the database is the genuine article, injected into db.server. +import { boundedIn } from "@trigger.dev/database"; +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; + +vi.setConfig({ testTimeout: 60_000 }); + +const db = vi.hoisted(() => ({ client: null as unknown as PrismaClient })); + +vi.mock("~/services/routeBuilders/dashboardBuilder", () => ({ + dashboardAction: (_options: unknown, handler: unknown) => handler, + dashboardLoader: (_options: unknown, handler: unknown) => handler, +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return db.client; + }, + boundedIn, +})); + +import { action } from "~/routes/admin.feature-flags"; + +const WORKER_GROUP_ID = "clwg000000000000000000000"; + +async function post(host: string, body: unknown) { + const request = new Request(`https://${host}/admin/feature-flags`, { + method: "POST", + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + }); + return (await (action as any)({ request, params: {}, context: {} })) as Response; +} + +async function readFlag(prisma: PrismaClient, key: string) { + const row = await prisma.featureFlag.findFirst({ where: { key }, select: { value: true } }); + return row?.value; +} + +async function seed(prisma: PrismaClient) { + db.client = prisma; + await prisma.featureFlag.createMany({ + data: [ + { id: "ff_locked", key: FEATURE_FLAG.defaultWorkerInstanceGroupId, value: WORKER_GROUP_ID }, + { id: "ff_plain", key: FEATURE_FLAG.mollifierEnabled, value: true }, + ], + }); +} + +describe("admin feature flags action", () => { + postgresTest("keeps the locked flag when the page did not unlock it", async ({ prisma }) => { + await seed(prisma); + + const response = await post("localhost:3030", { flags: {} }); + + expect(response.status).toBe(200); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + }); + + postgresTest("keeps the locked flag when the body omits the unlock field", async ({ prisma }) => { + await seed(prisma); + + // A tab opened before the field existed posts the old shape. + const response = await post("localhost:3030", { flags: {}, unlockLockedFlags: undefined }); + + expect(response.status).toBe(200); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + }); + + postgresTest("deletes the locked flag when the page unlocked it", async ({ prisma }) => { + await seed(prisma); + + await post("localhost:3030", { flags: {}, unlockLockedFlags: true }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest( + "keeps the locked flag on managed cloud despite the unlock claim", + async ({ prisma }) => { + await seed(prisma); + + await post("cloud.trigger.dev", { flags: {}, unlockLockedFlags: true }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + } + ); + + postgresTest( + "rejects a locked flag submitted to managed cloud, writing nothing", + async ({ prisma }) => { + await seed(prisma); + + const response = await post("cloud.trigger.dev", { + flags: { [FEATURE_FLAG.defaultWorkerInstanceGroupId]: "clwg999" }, + }); + + expect(response.status).toBe(400); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + } + ); + + postgresTest("rejects a value the catalog refuses, writing nothing", async ({ prisma }) => { + await seed(prisma); + + const response = await post("localhost:3030", { + flags: { [FEATURE_FLAG.realtimeBackend]: "not-a-backend" }, + }); + + expect(response.status).toBe(400); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + postgresTest("upserts what was submitted and sweeps what was not", async ({ prisma }) => { + await seed(prisma); + + await post("localhost:3030", { + flags: { [FEATURE_FLAG.hasAiAccess]: true }, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBe(true); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + }); +}); diff --git a/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts b/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts new file mode 100644 index 00000000000..ce18ef800e3 --- /dev/null +++ b/apps/webapp/test/globalFeatureFlagsLockedFlags.test.ts @@ -0,0 +1,150 @@ +// With "Unlock read-only flags" off, the page strips GLOBAL_LOCKED_FLAGS from its payload, so an +// omitted locked key means "the UI never offered it", not "the admin unset it". +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG, FeatureFlagCatalog, type FeatureFlagKey } from "~/v3/featureFlags"; +import { makeSetMultipleFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const CATALOG_KEYS = Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]; +const WORKER_GROUP_ID = "clwg000000000000000000000"; + +async function readFlag(prisma: PrismaClient, key: FeatureFlagKey): Promise { + const row = await prisma.featureFlag.findFirst({ where: { key }, select: { value: true } }); + return row?.value; +} + +describe("replaceGlobalFeatureFlags — locked flags the UI never submitted", () => { + postgresTest( + "keeps defaultWorkerInstanceGroupId when a locked flag is absent from the payload", + async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + [FEATURE_FLAG.mollifierEnabled]: true, + }); + + // What the page posts when an admin unsets mollifierEnabled on a self-hosted instance. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBeUndefined(); + } + ); + + postgresTest("an unlocked self-hosted page can still unset a locked flag", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest( + "managed cloud keeps locked flags even when unlocking is claimed", + async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isManagedCloud: true, + unlockLockedFlags: true, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe( + WORKER_GROUP_ID + ); + } + ); + + postgresTest("managed cloud still sweeps ordinary flags it was not sent", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + [FEATURE_FLAG.hasAiAccess]: true, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.mollifierEnabled]: true }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: true, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBe(WORKER_GROUP_ID); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBeUndefined(); + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + // The upsert and the sweep share one statement, which is only safe while no key is in both. + postgresTest("a submitted key is never also swept", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.mollifierEnabled]: true, + [FEATURE_FLAG.hasAiAccess]: true, + [FEATURE_FLAG.defaultWorkerInstanceGroupId]: WORKER_GROUP_ID, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.mollifierEnabled]: false, + [FEATURE_FLAG.hasAiAccess]: true, + }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + }); + + // Both submitted keys survive with their new values rather than being swept by the same + // statement that wrote them. + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(false); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBe(true); + expect(await readFlag(prisma, FEATURE_FLAG.defaultWorkerInstanceGroupId)).toBeUndefined(); + }); + + postgresTest("writes nothing when there is nothing to write", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: [], + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(true); + }); + + postgresTest("submitted flags are upserted and omitted ones swept", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.mollifierEnabled]: true, + [FEATURE_FLAG.hasAiAccess]: true, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.mollifierEnabled]: false }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: false, + }); + + expect(await readFlag(prisma, FEATURE_FLAG.mollifierEnabled)).toBe(false); + expect(await readFlag(prisma, FEATURE_FLAG.hasAiAccess)).toBeUndefined(); + }); +}); From 0205feda393c6c05be44a20c940a59b46d1fd116 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:22:33 +0100 Subject: [PATCH 02/28] refactor(run-engine): extract a WaitpointCoordinator seam around the Postgres waitpoint implementation (#4753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts every Postgres waitpoint and edge operation out of `WaitpointSystem` into a `WaitpointCoordinator` seam with one Postgres implementation, so a different coordination backend can be plugged in later without any caller changing. Pure refactor. Zero behaviour change, and zero test-file diffs — the existing engine corpus is the characterisation test. ## What moved `WaitpointCoordinator` (`waitpointCoordinator/types.ts`, declared with `type`) has nine members: `clearRunBlockState`, `readRunBlockState`, `registerBlocks`, `registerBlocksLockless`, `complete`, `createDateTimeWaitpoint`, `createManualWaitpoint`, `mintAssociatedWaitpointData`, `createAssociatedWaitpoint`. `LegacyPostgresWaitpointCoordinator` implements them against the run-ops store. Its dependencies are `{ runStore, prisma, logger }` only, so it structurally cannot reach the run lock, the worker, or the event bus — orchestration stays in `WaitpointSystem`, which keeps all ten public signatures, all six `worker.enqueue` sites, the racepoints, the snapshot transitions, and the event emissions. Two register methods rather than one with a flag, so "the batch path issues no extra query" is structural instead of conditional. Both share one private edge-write helper. ## Six notes for reviewers — please read before "simplifying" any of these 1. **`nanoid(24)` is called twice with different values on purpose**, in each create path: once for the upsert `where` key, once for `create.data`. Hoisting either to a shared constant makes the where-key match the create-key, turning a guaranteed-miss upsert into a possible update. In `createManualWaitpoint` both calls plus `WaitpointId.generate()` stay *inside* the retry loop so each attempt tries a fresh key. 2. **The two enqueue conditions are deliberately asymmetric.** DATETIME enqueues `finishWaitpoint` unconditionally after a non-cached create, with `availableAt: completedAfter`. MANUAL enqueues only when `timeout` is set. That is existing behaviour, not an oversight. The coordinator returns a discriminated union on `kind` rather than a boolean so the enqueue is structurally unreachable on the cached path. 3. **One false clause was deleted from a moved comment.** The old comment on the full-clear delete claimed the caller's `tx` is not forwarded. The code does forward it, and `PostgresRunStore` uses `tx ?? this.prisma`, so a single store joins the caller's transaction — only the routing store strips it. The rest of that comment is unchanged. 4. **The MANUAL timeout enqueue now sits outside the P2002 retry loop.** Safe because the worker is Redis-backed and cannot raise `Prisma.PrismaClientKnownRequestError`, so the loop never retried on it. **If a Postgres-backed enqueue is ever swapped in, that equivalence breaks silently.** 5. **The coordinator caches `runStore`/`prisma`/`logger` at construction**, where the old code read `this.$.*` per call. Equivalent only because nothing reassigns them: one assignment at `engine/index.ts`, and the `resources` object is a `const` that is never mutated. 6. **Two comments in other files are now stale and were left alone** — `engine/index.ts` and `completeWaitpointCrossSeamGuard.test.ts` both describe routing as the first statement of `waitpointSystem.completeWaitpoint`. Both tests still pass, because that guard sits in `index.ts` before the delegation. Left untouched to keep this diff to three files. ## Preserved verbatim The `unnest` edge CTE rather than a `Waitpoint` join; the pending count as a separate statement after the edge write (READ COMMITTED needs its own snapshot); completion's `findWaitpointOnPrimary` re-read through the *resolved handle* while the blocked-run fan-out goes back through the *router*; the residency and colocate hints, with colocation objects built only in the Postgres arm and the count keeping its `runId` argument; `ON CONFLICT DO NOTHING` and the `(taskRunId, waitpointId, batchIndex)` multi-index edge semantics; the unread `batchId` select, which rides inside two `logger.debug` payloads. `internal-packages/run-store/` is untouched, so the CTE and the conflict semantics never moved. ## Verification | Check | Result | | --- | --- | | Engine corpus | 61/61 files, 353 passed, 1 skipped, **0 failed** (baseline: 352 passed, 1 failed) | | Test-file diffs | **empty** | | `run-engine` typecheck | `tsc --noEmit -p tsconfig.build.json` exits 0 | | `webapp` typecheck | 146 errors on this branch, **146 identical errors at baseline** — pre-existing, none added | The webapp typecheck does not pass. The failures are pre-existing (`PrismaPg` not assignable to `never`; missing `@trigger.dev/rbac` exports) and the sorted error lists are byte-identical to the merge base, so this branch adds none — but the criterion is genuinely unmet and needs a separate fix. No changeset and no `.server-changes` note: internal refactor with no user-visible change. ## Follow-ups this surfaced - The dominant RUN waitpoint is still created outside the seam — `buildRunAssociatedWaitpoint` now mints through the coordinator, but the row is inserted nested inside `createRun`/`createFailedRun`. That needs its own packet before a second backend lands, or the commonest waitpoint gets split across two of them. - `clearRunBlockState` overloads opposite outcomes on `undefined` versus `[]`: `undefined` clears every edge, `[]` clears none. Both callers are correct today; worth splitting when the file is next touched. - A stray non-`.sql` entry in `internal-packages/clickhouse/schema/` breaks every `containerTest` in the repo, because the testcontainers migration reader `readFile`s every `readdir` entry without filtering despite a comment claiming it filters. Hit this during setup; unrelated to this change and left for a separate fix. --- .../src/engine/systems/waitpointSystem.ts | 377 +++------------ .../legacyPostgresCoordinator.ts | 437 ++++++++++++++++++ .../src/engine/waitpointCoordinator/types.ts | 145 ++++++ 3 files changed, 651 insertions(+), 308 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index 5d5a80772a6..3dbed999445 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -1,5 +1,4 @@ -import { timeoutError, tryCatch } from "@trigger.dev/core/v3"; -import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { timeoutError } from "@trigger.dev/core/v3"; import type { PrismaClientOrTransaction, TaskRun, @@ -7,13 +6,11 @@ import type { TaskRunExecutionStatus, Waitpoint, } from "@trigger.dev/database"; -import { Prisma, boundedIn } from "@trigger.dev/database"; -import type { RunStore } from "@internal/run-store"; import { assertNever } from "assert-never"; -import { nanoid } from "nanoid"; -import { UnclassifiableWaitpointId } from "../errors.js"; import { sendNotificationToWorker } from "../eventBus.js"; import { isFinalRunStatus } from "../statuses.js"; +import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js"; +import type { WaitpointCoordinator } from "../waitpointCoordinator/types.js"; import type { EnqueueSystem } from "./enqueueSystem.js"; import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js"; import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js"; @@ -45,11 +42,17 @@ export class WaitpointSystem { private readonly $: SystemResources; private readonly executionSnapshotSystem: ExecutionSnapshotSystem; private readonly enqueueSystem: EnqueueSystem; + private readonly coordinator: WaitpointCoordinator; constructor(private readonly options: WaitpointSystemOptions) { this.$ = options.resources; this.executionSnapshotSystem = options.executionSnapshotSystem; this.enqueueSystem = options.enqueueSystem; + this.coordinator = new LegacyPostgresWaitpointCoordinator({ + runStore: this.$.runStore, + prisma: this.$.prisma, + logger: this.$.logger, + }); } public async clearBlockingWaitpoints({ @@ -59,14 +62,7 @@ export class WaitpointSystem { runId: string; tx?: PrismaClientOrTransaction; }) { - // A run's edges co-locate with the run (the edge write routes by runId), so the router routes this - // taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is not - // forwarded — the delete runs on the owning store's own client (the router never threads a - // control-plane tx into a routed write). - const deleted = await this.$.runStore.deleteManyTaskRunWaitpoints( - { where: { taskRunId: runId } }, - tx - ); + const deleted = await this.coordinator.clearRunBlockState({ runId, tx }); return deleted.count; } @@ -84,86 +80,19 @@ export class WaitpointSystem { isError: boolean; }; }): Promise { - // Residency store-selection guard. completeWaitpoint arrives with only - // (waitpointId, output) — no run id — so the owning run-ops store is selected - // by the waitpoint's own residency. In single-DB this is the one store - // (no classification). An unclassifiable id throws loud — never default-routes. - let store: RunStore; - try { - store = await this.$.runStore.forWaitpointCompletion(id, { routeKind: "MANUAL" }); - } catch (error) { - this.$.logger.error("completeWaitpoint: unclassifiable waitpointId", { - waitpointId: id, - error, - }); - throw new UnclassifiableWaitpointId(id, { cause: error }); - } - - // 1. Complete the Waitpoint (if not completed) - const [updateError, updateResult] = await tryCatch( - store.updateManyWaitpoints({ - where: { id, status: "PENDING" }, - data: { - status: "COMPLETED", - completedAt: new Date(), - output: output?.value, - outputType: output?.type, - outputIsError: output?.isError, - }, - }) - ); - - if (updateError) { - this.$.logger.error("completeWaitpoint: error updating waitpoint:", { updateError }); - throw updateError; - } - - if (updateResult.count === 0) { - this.$.logger.info( - "completeWaitpoint: attempted to complete a waitpoint that is not PENDING", - { waitpointId: id } - ); - } - - // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's - // default) can miss it under lag → false "not found" → the parent hangs; this.$.prisma would - // instead hit the wrong DB. findWaitpointOnPrimary reads the owning store's primary. - const waitpoint = await store.findWaitpointOnPrimary({ - where: { id }, + const { waitpoint, blockedRuns } = await this.coordinator.complete({ + waitpointId: id, + output, }); - if (!waitpoint) { - this.$.logger.error("completeWaitpoint: waitpoint not found", { waitpointId: id }); - throw new Error("Waitpoint not found"); - } - - if (waitpoint.status !== "COMPLETED") { - this.$.logger.error(`completeWaitpoint: waitpoint is not completed`, { - waitpointId: id, - }); - throw new Error("Waitpoint not completed"); - } - - // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates - // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router - // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`, - // or a cross-DB blocked run is never found and hangs forever. - const affectedTaskRuns = await this.$.runStore.findManyTaskRunWaitpoints( - { - where: { waitpointId: id }, - select: { taskRunId: true, spanIdToComplete: true, createdAt: true }, - }, - this.$.prisma - ); - - if (affectedTaskRuns.length === 0) { + if (blockedRuns.length === 0) { this.$.logger.debug(`completeWaitpoint: no TaskRunWaitpoints found for waitpoint`, { waitpointId: id, }); } // 3. Schedule trying to continue the runs - for (const run of affectedTaskRuns) { + for (const run of blockedRuns) { const jobId = `continueRunIfUnblocked:${run.taskRunId}`; //50ms in the future const availableAt = new Date(Date.now() + 50); @@ -220,81 +149,27 @@ export class WaitpointSystem { idempotencyKey?: string; idempotencyKeyExpiresAt?: Date; }) { - // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that - // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay - // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert - // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup - // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the - // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to - // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the - // run store (never a caller tx) so it can never bypass residency onto the wrong DB. - const colocate = runId ? { coLocateWithRunId: runId } : undefined; - const existingWaitpoint = idempotencyKey - ? await this.$.runStore.findWaitpoint( - { - where: { - environmentId, - idempotencyKey, - }, - }, - undefined, - colocate - ) - : undefined; - - if (existingWaitpoint) { - if ( - existingWaitpoint.idempotencyKeyExpiresAt && - new Date() > existingWaitpoint.idempotencyKeyExpiresAt - ) { - //the idempotency key has expired - //remove the waitpoint idempotencyKey - const rotateArgs = { - where: { - id: existingWaitpoint.id, - }, - data: { - idempotencyKey: nanoid(24), - inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, - }, - }; - await this.$.runStore.updateWaitpoint(rotateArgs, undefined, colocate); + const result = await this.coordinator.createDateTimeWaitpoint({ + runId, + projectId, + environmentId, + completedAfter, + idempotencyKey, + idempotencyKeyExpiresAt, + }); - //let it fall through to create a new waitpoint - } else { - return { waitpoint: existingWaitpoint, isCached: true }; - } + if (result.kind === "cached") { + return { waitpoint: result.waitpoint, isCached: true }; } - const upsertArgs = { - where: { - environmentId_idempotencyKey: { - environmentId, - idempotencyKey: idempotencyKey ?? nanoid(24), - }, - }, - create: { - ...WaitpointId.generate(), - type: "DATETIME" as const, - idempotencyKey: idempotencyKey ?? nanoid(24), - idempotencyKeyExpiresAt, - userProvidedIdempotencyKey: !!idempotencyKey, - environmentId, - projectId, - completedAfter, - }, - update: {}, - }; - const waitpoint = await this.$.runStore.upsertWaitpoint(upsertArgs, undefined, colocate); - await this.$.worker.enqueue({ - id: `finishWaitpoint.${waitpoint.id}`, + id: `finishWaitpoint.${result.waitpoint.id}`, job: "finishWaitpoint", - payload: { waitpointId: waitpoint.id }, + payload: { waitpointId: result.waitpoint.id }, availableAt: completedAfter, }); - return { waitpoint, isCached: false }; + return { waitpoint: result.waitpoint, isCached: false }; } /** This creates a MANUAL waitpoint, that can be explicitly completed (or failed). @@ -322,117 +197,35 @@ export class WaitpointSystem { // to LEGACY by its cuid id-shape. Ignored when `runId` is set (co-location wins). standaloneResidency?: "NEW" | "LEGACY"; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { - // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint - // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A - // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an - // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the - // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via - // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here. - const colocate = runId - ? { coLocateWithRunId: runId } - : standaloneResidency - ? { residency: standaloneResidency } - : undefined; - const existingWaitpoint = idempotencyKey - ? await this.$.runStore.findWaitpoint( - { - where: { - environmentId, - idempotencyKey, - }, - }, - undefined, - colocate - ) - : undefined; - - if (existingWaitpoint) { - if ( - existingWaitpoint.idempotencyKeyExpiresAt && - new Date() > existingWaitpoint.idempotencyKeyExpiresAt - ) { - //the idempotency key has expired - //remove the waitpoint idempotencyKey - await this.$.runStore.updateWaitpoint( - { - where: { - id: existingWaitpoint.id, - }, - data: { - idempotencyKey: nanoid(24), - inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, - }, - }, - undefined, - colocate - ); + const result = await this.coordinator.createManualWaitpoint({ + runId, + environmentId, + projectId, + idempotencyKey, + idempotencyKeyExpiresAt, + timeout, + tags, + standaloneResidency, + }); - //let it fall through to create a new waitpoint - } else { - return { waitpoint: existingWaitpoint, isCached: true }; - } + if (result.kind === "cached") { + return { waitpoint: result.waitpoint, isCached: true }; } - const maxRetries = 5; - let attempts = 0; - - while (attempts < maxRetries) { - try { - const waitpoint = await this.$.runStore.upsertWaitpoint( - { - where: { - environmentId_idempotencyKey: { - environmentId, - idempotencyKey: idempotencyKey ?? nanoid(24), - }, - }, - create: { - ...WaitpointId.generate(), - type: "MANUAL", - idempotencyKey: idempotencyKey ?? nanoid(24), - idempotencyKeyExpiresAt, - userProvidedIdempotencyKey: !!idempotencyKey, - environmentId, - projectId, - completedAfter: timeout, - tags, - }, - update: {}, - }, - undefined, - colocate - ); - - //schedule the timeout - if (timeout) { - await this.$.worker.enqueue({ - id: `finishWaitpoint.${waitpoint.id}`, - job: "finishWaitpoint", - payload: { - waitpointId: waitpoint.id, - error: JSON.stringify(timeoutError(timeout)), - }, - availableAt: timeout, - }); - } - - return { waitpoint, isCached: false }; - } catch (error) { - if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { - // Handle unique constraint violation (conflict) - attempts++; - if (attempts >= maxRetries) { - throw new Error( - `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.` - ); - } - } else { - throw error; // Re-throw other errors - } - } + //schedule the timeout + if (timeout) { + await this.$.worker.enqueue({ + id: `finishWaitpoint.${result.waitpoint.id}`, + job: "finishWaitpoint", + payload: { + waitpointId: result.waitpoint.id, + error: JSON.stringify(timeoutError(timeout)), + }, + availableAt: timeout, + }); } - throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`); + return { waitpoint: result.waitpoint, isCached: false }; } /** @@ -489,25 +282,19 @@ export class WaitpointSystem { this.$.runStore ); - // Insert the blocking + historical connections via the run-ops store, routed by the owning - // run id so the edge co-resides with the run. Never pinned to the caller's control-plane tx: - // that joined `Waitpoint` on the wrong DB and wrote 0 edges. The pending check stays a - // SEPARATE store call so it gets its own READ COMMITTED snapshot (see the doc comment above). - await this.$.runStore.blockRunWithWaitpointEdges({ + // Insert the blocking + historical connections and re-check the pending count. The + // coordinator keeps these as two separate store statements, in this order, for the READ + // COMMITTED reason documented on the method and in the doc comment above. + const { pendingCount } = await this.coordinator.registerBlocks({ runId, waitpointIds: $waitpoints, projectId, spanIdToComplete, batchId: batch?.id, batchIndex: batch?.index, + client: prisma, }); - // Check if the run is actually blocked using a separate query (see above). Pass the writer so the - // pending re-read is read-your-writes on the owning PRIMARY (a lagging replica can strand the run). - // Route by the blocked run id: its blocking waitpoints co-locate with the run, so the router - // counts on the run's store and only falls back to the other DB for a cross-tree token. - const pendingCount = await this.$.runStore.countPendingWaitpoints($waitpoints, prisma, runId); - const isRunBlocked = pendingCount > 0; let newStatus: TaskRunExecutionStatus = "SUSPENDED"; @@ -605,10 +392,10 @@ export class WaitpointSystem { }): Promise { const $waitpoints = typeof waitpoints === "string" ? [waitpoints] : waitpoints; - // Same routed edge write as blockRunWithWaitpoint, routed by the owning run id. No lock - // needed: ON CONFLICT DO NOTHING makes concurrent inserts safe, and the parent snapshot is - // already EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch. - await this.$.runStore.blockRunWithWaitpointEdges({ + // Same routed edge write as blockRunWithWaitpoint. No lock needed: ON CONFLICT DO NOTHING + // makes concurrent inserts safe, and the parent snapshot is already + // EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch. No pending count here. + await this.coordinator.registerBlocksLockless({ runId, waitpointIds: $waitpoints, projectId, @@ -682,20 +469,7 @@ export class WaitpointSystem { return await this.$.runLock.lock("continueRunIfUnblocked", [runId], async () => { // 1. Get the any blocking waitpoints - const blockingWaitpoints = await this.$.runStore.findManyTaskRunWaitpoints( - { - where: { taskRunId: runId }, - select: { - id: true, - batchId: true, - batchIndex: true, - waitpoint: { - select: { id: true, status: true, type: true, completedAfter: true }, - }, - }, - }, - this.$.prisma - ); + const blockingWaitpoints = await this.coordinator.readRunBlockState(runId); // 2. There are blockers still, so do nothing if (blockingWaitpoints.some((w) => w.waitpoint.status !== "COMPLETED")) { @@ -926,11 +700,9 @@ export class WaitpointSystem { if (blockingWaitpoints.length > 0) { //5. Remove the blocking waitpoints - await this.$.runStore.deleteManyTaskRunWaitpoints({ - where: { - taskRunId: runId, - id: { in: boundedIn(blockingWaitpoints.map((b) => b.id)) }, - }, + await this.coordinator.clearRunBlockState({ + runId, + edgeIds: blockingWaitpoints.map((b) => b.id), }); this.$.logger.debug(`continueRunIfUnblocked: removed blocking waitpoints`, { @@ -953,15 +725,7 @@ export class WaitpointSystem { projectId: string; environmentId: string; }) { - return { - ...WaitpointId.generate(), - type: "RUN" as const, - status: "PENDING" as const, - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - environmentId, - }; + return this.coordinator.mintAssociatedWaitpointData({ projectId, environmentId }); } /** @@ -1045,12 +809,9 @@ export class WaitpointSystem { // Create waitpoint and link to run atomically const waitpointData = this.buildRunAssociatedWaitpoint({ projectId, environmentId }); - // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id. - const waitpoint = await this.$.runStore.createWaitpoint({ - data: { - ...waitpointData, - completedByTaskRunId: runId, - }, + const waitpoint = await this.coordinator.createAssociatedWaitpoint({ + runId, + data: waitpointData, }); // If run has already finished (per snapshot), complete the waitpoint immediately so the parent can resume diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts new file mode 100644 index 00000000000..d1e48fa4f8d --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -0,0 +1,437 @@ +import type { RunStore } from "@internal/run-store"; +import { tryCatch } from "@trigger.dev/core/v3"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import type { Logger } from "@trigger.dev/core/logger"; +import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; +import { boundedIn, Prisma } from "@trigger.dev/database"; +import { nanoid } from "nanoid"; +import { UnclassifiableWaitpointId } from "../errors.js"; +import type { + AssociatedWaitpointData, + ClearRunBlockStateParams, + CompleteParams, + CompleteResult, + CreateDateTimeWaitpointParams, + CreateManualWaitpointParams, + CreateWaitpointResult, + RegisterBlocksLocklessParams, + RegisterBlocksParams, + RunBlockEdge, + WaitpointCoordinator, +} from "./types.js"; + +export type LegacyPostgresWaitpointCoordinatorOptions = { + runStore: RunStore; + prisma: PrismaClient; + logger: Logger; +}; + +/** + * Waitpoint coordination against Postgres, through the run-ops store. + * + * Dependencies are deliberately narrow: no run lock, no worker, no event bus. + * That makes "this owns waitpoint state only" structural rather than a convention. + */ +export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator { + private readonly runStore: RunStore; + private readonly prisma: PrismaClient; + private readonly logger: Logger; + + constructor(options: LegacyPostgresWaitpointCoordinatorOptions) { + this.runStore = options.runStore; + this.prisma = options.prisma; + this.logger = options.logger; + } + + async clearRunBlockState({ + runId, + edgeIds, + tx, + }: ClearRunBlockStateParams): Promise<{ count: number }> { + if (edgeIds) { + // Bounded delete of named edges, on the unblock path. No tx: that path is not inside a + // caller transaction, and boundedIn caps the id-list arity for Prisma. + return this.runStore.deleteManyTaskRunWaitpoints({ + where: { + taskRunId: runId, + id: { in: boundedIn(edgeIds) }, + }, + }); + } + + // A run's edges co-locate with the run (the edge write routes by runId), so the router routes + // this taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is + // passed through: a routing store strips it, and a single store joins it. + return this.runStore.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx); + } + + async readRunBlockState(runId: string): Promise { + return this.runStore.findManyTaskRunWaitpoints( + { + where: { taskRunId: runId }, + select: { + id: true, + batchId: true, + batchIndex: true, + waitpoint: { + select: { id: true, status: true, type: true, completedAfter: true }, + }, + }, + }, + this.prisma + ); + } + + async registerBlocks({ + client, + ...edge + }: RegisterBlocksParams): Promise<{ pendingCount: number }> { + await this.#writeBlockEdges(edge); + + // Check if the run is actually blocked using a separate query. The separate statement is the + // point: under PostgreSQL READ COMMITTED each statement gets its own snapshot, so a + // concurrent completion that commits between the edge write and this check is still seen. + // It queries ALL requested ids, not just inserted ones: a row that already existed (ON + // CONFLICT skipped the insert) but is still PENDING must still block. Pass the caller's + // client so the re-read is read-your-writes on the owning PRIMARY, and pass the run id so + // the router counts on the run's store instead of fanning out to both DBs. + const pendingCount = await this.runStore.countPendingWaitpoints( + edge.waitpointIds, + client, + edge.runId + ); + + return { pendingCount }; + } + + async registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise { + await this.#writeBlockEdges(params); + } + + async complete({ waitpointId, output }: CompleteParams): Promise { + // Residency store-selection guard. complete arrives with only (waitpointId, output) — no run + // id — so the owning run-ops store is selected by the waitpoint's own residency. In single-DB + // this is the one store (no classification). An unclassifiable id throws loud — never + // default-routes. The try wraps ONLY the resolve: widening it would swallow the + // "Waitpoint not found" path that a single store relies on. + let store: RunStore; + try { + store = await this.runStore.forWaitpointCompletion(waitpointId, { routeKind: "MANUAL" }); + } catch (error) { + this.logger.error("completeWaitpoint: unclassifiable waitpointId", { + waitpointId, + error, + }); + throw new UnclassifiableWaitpointId(waitpointId, { cause: error }); + } + + // 1. Complete the Waitpoint (if not completed) + const [updateError, updateResult] = await tryCatch( + store.updateManyWaitpoints({ + where: { id: waitpointId, status: "PENDING" }, + data: { + status: "COMPLETED", + completedAt: new Date(), + output: output?.value, + outputType: output?.type, + outputIsError: output?.isError, + }, + }) + ); + + if (updateError) { + this.logger.error("completeWaitpoint: error updating waitpoint:", { updateError }); + throw updateError; + } + + if (updateResult.count === 0) { + this.logger.info("completeWaitpoint: attempted to complete a waitpoint that is not PENDING", { + waitpointId, + }); + } + + // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's + // default) can miss it under lag → false "not found" → the parent hangs. Going back through + // the router would re-resolve the store and change the routing, so use the handle. + const waitpoint = await store.findWaitpointOnPrimary({ + where: { id: waitpointId }, + }); + + if (!waitpoint) { + this.logger.error("completeWaitpoint: waitpoint not found", { waitpointId }); + throw new Error("Waitpoint not found"); + } + + if (waitpoint.status !== "COMPLETED") { + this.logger.error(`completeWaitpoint: waitpoint is not completed`, { waitpointId }); + throw new Error("Waitpoint not completed"); + } + + // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates + // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router + // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`, + // or a cross-DB blocked run is never found and hangs forever. + const blockedRuns = await this.runStore.findManyTaskRunWaitpoints( + { + where: { waitpointId }, + select: { taskRunId: true, spanIdToComplete: true, createdAt: true }, + }, + this.prisma + ); + + return { waitpoint, blockedRuns }; + } + + async createDateTimeWaitpoint({ + runId, + projectId, + environmentId, + completedAfter, + idempotencyKey, + idempotencyKeyExpiresAt, + }: CreateDateTimeWaitpointParams): Promise { + // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that + // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay + // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert + // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup + // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the + // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to + // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the + // run store (never a caller tx) so it can never bypass residency onto the wrong DB. + const colocate = runId ? { coLocateWithRunId: runId } : undefined; + const existingWaitpoint = idempotencyKey + ? await this.runStore.findWaitpoint( + { + where: { + environmentId, + idempotencyKey, + }, + }, + undefined, + colocate + ) + : undefined; + + if (existingWaitpoint) { + if ( + existingWaitpoint.idempotencyKeyExpiresAt && + new Date() > existingWaitpoint.idempotencyKeyExpiresAt + ) { + //the idempotency key has expired + //remove the waitpoint idempotencyKey + const rotateArgs = { + where: { + id: existingWaitpoint.id, + }, + data: { + idempotencyKey: nanoid(24), + inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, + }, + }; + await this.runStore.updateWaitpoint(rotateArgs, undefined, colocate); + + //let it fall through to create a new waitpoint + } else { + return { kind: "cached", waitpoint: existingWaitpoint }; + } + } + + // The two `nanoid(24)` calls below are deliberately separate and produce DIFFERENT values: + // the upsert `where` key must not match the `create` key, or a guaranteed-miss upsert becomes + // a possible update. Do not hoist either to a shared constant. + const upsertArgs = { + where: { + environmentId_idempotencyKey: { + environmentId, + idempotencyKey: idempotencyKey ?? nanoid(24), + }, + }, + create: { + ...WaitpointId.generate(), + type: "DATETIME" as const, + idempotencyKey: idempotencyKey ?? nanoid(24), + idempotencyKeyExpiresAt, + userProvidedIdempotencyKey: !!idempotencyKey, + environmentId, + projectId, + completedAfter, + }, + update: {}, + }; + const waitpoint = await this.runStore.upsertWaitpoint(upsertArgs, undefined, colocate); + + return { kind: "created", waitpoint }; + } + + async createManualWaitpoint({ + runId, + environmentId, + projectId, + idempotencyKey, + idempotencyKeyExpiresAt, + timeout, + tags, + standaloneResidency, + }: CreateManualWaitpointParams): Promise { + // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint + // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A + // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an + // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the + // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via + // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here. + const colocate = runId + ? { coLocateWithRunId: runId } + : standaloneResidency + ? { residency: standaloneResidency } + : undefined; + const existingWaitpoint = idempotencyKey + ? await this.runStore.findWaitpoint( + { + where: { + environmentId, + idempotencyKey, + }, + }, + undefined, + colocate + ) + : undefined; + + if (existingWaitpoint) { + if ( + existingWaitpoint.idempotencyKeyExpiresAt && + new Date() > existingWaitpoint.idempotencyKeyExpiresAt + ) { + //the idempotency key has expired + //remove the waitpoint idempotencyKey + await this.runStore.updateWaitpoint( + { + where: { + id: existingWaitpoint.id, + }, + data: { + idempotencyKey: nanoid(24), + inactiveIdempotencyKey: existingWaitpoint.idempotencyKey, + }, + }, + undefined, + colocate + ); + + //let it fall through to create a new waitpoint + } else { + return { kind: "cached", waitpoint: existingWaitpoint }; + } + } + + const maxRetries = 5; + let attempts = 0; + + while (attempts < maxRetries) { + try { + // As in createDateTimeWaitpoint, the two `nanoid(24)` calls are deliberately separate and + // differ. Both, and `WaitpointId.generate()`, are re-evaluated on every attempt: that is + // what makes a retry after a unique-constraint conflict try a fresh key. + const waitpoint = await this.runStore.upsertWaitpoint( + { + where: { + environmentId_idempotencyKey: { + environmentId, + idempotencyKey: idempotencyKey ?? nanoid(24), + }, + }, + create: { + ...WaitpointId.generate(), + type: "MANUAL", + idempotencyKey: idempotencyKey ?? nanoid(24), + idempotencyKeyExpiresAt, + userProvidedIdempotencyKey: !!idempotencyKey, + environmentId, + projectId, + completedAfter: timeout, + tags, + }, + update: {}, + }, + undefined, + colocate + ); + + return { kind: "created", waitpoint }; + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + // Handle unique constraint violation (conflict) + attempts++; + if (attempts >= maxRetries) { + throw new Error( + `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.` + ); + } + } else { + throw error; // Re-throw other errors + } + } + } + + throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`); + } + + mintAssociatedWaitpointData({ + projectId, + environmentId, + }: { + projectId: string; + environmentId: string; + }): AssociatedWaitpointData { + return { + ...WaitpointId.generate(), + type: "RUN" as const, + status: "PENDING" as const, + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + environmentId, + }; + } + + async createAssociatedWaitpoint({ + runId, + data, + }: { + runId: string; + data: AssociatedWaitpointData; + }): Promise { + // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id. + return this.runStore.createWaitpoint({ + data: { + ...data, + completedByTaskRunId: runId, + }, + }); + } + + /** + * The edge write, shared by both register paths so they cannot drift. + * + * Routed by the owning run id so the edge co-resides with the run. Never pinned to a caller + * transaction: that joined `Waitpoint` on the wrong DB, wrote 0 edges, and silently never + * suspended the parent. The write is idempotent (ON CONFLICT DO NOTHING). + */ + #writeBlockEdges({ + runId, + waitpointIds, + projectId, + spanIdToComplete, + batchId, + batchIndex, + }: RegisterBlocksLocklessParams): Promise { + return this.runStore.blockRunWithWaitpointEdges({ + runId, + waitpointIds, + projectId, + spanIdToComplete, + batchId, + batchIndex, + }); + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts new file mode 100644 index 00000000000..8a50abb7d1c --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -0,0 +1,145 @@ +import type { ReadClient } from "@internal/run-store"; +import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; + +/** + * The waitpoint and edge state operations that `WaitpointSystem` delegates. + * + * Orchestration stays in `WaitpointSystem`: the run lock, snapshot transitions, + * worker-job enqueues, event emissions and racepoints. This owns waitpoint and + * edge state only, so a non-Postgres implementation can replace it without any + * caller learning that it changed. + * + * The residency hints and `tx` are opaque pass-throughs. Opaque does not mean + * type-free — a Prisma type appears here — it means a non-Postgres implementation + * never reads the value. + */ +export type WaitpointCoordinator = { + clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>; + readRunBlockState(runId: string): Promise; + registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>; + registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise; + complete(params: CompleteParams): Promise; + createDateTimeWaitpoint(params: CreateDateTimeWaitpointParams): Promise; + createManualWaitpoint(params: CreateManualWaitpointParams): Promise; + mintAssociatedWaitpointData(params: { + projectId: string; + environmentId: string; + }): AssociatedWaitpointData; + createAssociatedWaitpoint(params: { + runId: string; + data: AssociatedWaitpointData; + }): Promise; +}; + +export type ClearRunBlockStateParams = { + runId: string; + /** Edge ids to delete. Omit to clear every edge for the run. */ + edgeIds?: string[]; + /** + * Forwarded verbatim on the full-clear leg only, and never on the bounded leg + * or an edge write. A routing store strips it; a single store joins it. + */ + tx?: PrismaClientOrTransaction; +}; + +/** + * One block edge, with the fields the unblock decision reads. + * + * `batchId` is read by no logic. It rides inside the two `logger.debug` payloads in + * `continueRunIfUnblocked`, so removing it changes log output. + */ +export type RunBlockEdge = { + id: string; + batchId: string | null; + batchIndex: number | null; + waitpoint: Pick; +}; + +export type RegisterBlocksParams = { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + /** + * Read client for the pending count only. The caller resolves `tx ?? prisma` once + * and passes the result, so the writer is used when the caller is inside a + * transaction and the pending re-read is read-your-writes on the owning primary. + * Never forwarded to the edge write. + */ + client: ReadClient; +}; + +/** + * The lockless variant writes the edge and does not count. Two methods rather than + * one method with a flag, so "the batch path issues no extra query" is structural. + */ +export type RegisterBlocksLocklessParams = Omit; + +export type CompleteParams = { + waitpointId: string; + output?: { + value: string; + type?: string; + isError: boolean; + }; +}; + +/** One run blocked by the completed waitpoint, with the fields the caller's fan-out loop reads. */ +type BlockedRun = { + taskRunId: string; + spanIdToComplete: string | null; + createdAt: Date; +}; + +export type CompleteResult = { + waitpoint: Waitpoint; + blockedRuns: BlockedRun[]; +}; + +/** + * Discriminated on purpose. The caller enqueues the `finishWaitpoint` job only in the + * `created` branch, because today's create methods return before their enqueue on the + * cached path. A boolean would let a later edit enqueue on both branches. + */ +export type CreateWaitpointResult = + | { kind: "cached"; waitpoint: Waitpoint } + | { kind: "created"; waitpoint: Waitpoint }; + +export type CreateDateTimeWaitpointParams = { + /** When set, the waitpoint co-locates with this run's DB and the dedup probe targets it. */ + runId?: string; + projectId: string; + environmentId: string; + completedAfter: Date; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; +}; + +export type CreateManualWaitpointParams = { + runId?: string; + environmentId: string; + projectId: string; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date; + timeout?: Date; + tags?: string[]; + /** + * See the `standaloneResidency` param doc on `WaitpointSystem.createManualWaitpoint` for the + * full rationale. Only a Postgres implementation reads this. + */ + standaloneResidency?: "NEW" | "LEGACY"; +}; + +/** The RUN-waitpoint row data. Pure — no store touch — so the mint is coordinator-owned. */ +export type AssociatedWaitpointData = { + id: string; + friendlyId: string; + type: "RUN"; + status: "PENDING"; + idempotencyKey: string; + userProvidedIdempotencyKey: false; + projectId: string; + environmentId: string; +}; From d6457521cba883d2916068140bc257b862ebefaa Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:43:08 +0100 Subject: [PATCH 03/28] fix(hosting): disable clickhouse system-log telemetry and apply profile settings via users.d (#4762) Carries over the self-hosted ClickHouse fix from #4546 by @Leafgard, whose commits are preserved here, plus follow-up polish. Opened in-repo because the fork is org-owned, which GitHub's "Allow edits from maintainers" doesn't cover. fixes #4343 ## What was wrong Two independent problems in `hosting/docker/clickhouse/`: 1. **The `` block never applied.** It sits in `override.xml`, mounted under `config.d` - but ClickHouse only reads profile settings from the users config tree. Verified on the pinned image: before this change `max_block_size` sat at its default `65409` with `changed=0`, so the advertised low-memory settings had never taken effect at all. 2. **Every ClickHouse system log table was enabled and unbounded.** On a sub-16GB machine their background merges outgrow the memory cap; ClickHouse's [low-RAM guide](https://clickhouse.com/docs/operations/tips) recommends disabling them. The dev stack already does this - `hosting/docker` never got it. ## What this does - `clickhouse/override.xml`: disables the high-frequency telemetry tables, and bounds the ones worth keeping with a config-level `` - `query_log` and `part_log` at 7 days, `error_log` at 30. A config-level TTL survives log-table recreation, unlike `ALTER ... MODIFY TTL`. - New `clickhouse/users-override.xml`, mounted at `users.d/override.xml`: carries the profile settings so they actually apply, completes the sub-16GB set with `max_threads=1`, and zeroes the memory/query profilers, whose samples were the main source feeding `trace_log`. - `webapp/docker-compose.yml`: adds the `users.d` mount. ## Verification Ran `clickhouse/clickhouse-server:26.2` with these exact mounts, and `25.12` to cover the documented 25.8 floor: - All 9 profile settings report `changed=1`, and a custom `CLICKHOUSE_USER` inherits them. - `users.d` merges rather than replaces: the `default` user, its password, `access_management` and the `readonly` profile all survive, so the compose healthcheck still passes. - `remove="1"` is a clean no-op on keys absent from a given version - no empty section, no accidental table, no startup error - so pinning `CLICKHOUSE_IMAGE_TAG` to an older supported tag won't crash-loop. - TTLs land in the real DDL: `TTL event_date + toIntervalDay(7)` / `(30)`. - In-place upgrade on a populated volume: clean restart, data preserved, and ClickHouse lazily renames the pre-existing `query_log`/`error_log` to `query_log_0`/`error_log_0` as it applies the new retention. ## Notes for review - **`part_log` is kept (bounded) rather than disabled.** It appears in neither report behind this change and isn't on ClickHouse's sub-16GB list, but it's the merge history you'd need to diagnose a recurrence. Measured at ~0.18 KiB per part event under insert churn - about 10x cheaper than `text_log` over the same window - so a TTL bounds it rather than removing it. - **The profile settings go live for the first time here.** On larger machines that's a real, intended throughput change: `max_threads=1`, `max_download_threads=1`, parallel parsing and formatting off. - **Disabling a log table stops new writes but doesn't delete existing data.** Reclaiming disk on an existing deployment needs `DROP TABLE system. SYNC`, including the `*_log_0` leftovers. ## Known gaps, deliberately not in this PR - The Helm chart carries the same ineffective `` block in `values.yaml` and mounts nothing into `users.d`, so this fix isn't currently expressible there. - `background_schedule_pool_log` is enabled by default with no TTL and is disabled by neither stack. - The dev stack's disable list has drifted from this one. - The compose healthcheck still logs a query every 5 seconds. --------- Co-authored-by: Yann SEGET Co-authored-by: Claude Fable 5 --- hosting/docker/clickhouse/override.xml | 39 +++++++++++++++----- hosting/docker/clickhouse/users-override.xml | 16 ++++++++ hosting/docker/webapp/docker-compose.yml | 1 + 3 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 hosting/docker/clickhouse/users-override.xml diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml index 41897c29849..253669ab09d 100644 --- a/hosting/docker/clickhouse/override.xml +++ b/hosting/docker/clickhouse/override.xml @@ -9,12 +9,33 @@ 524288000 1 - - - 8192 - 1 - 0 - 0 - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + event_date + INTERVAL 7 DAY DELETE + + + event_date + INTERVAL 7 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + + diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml new file mode 100644 index 00000000000..46699505e34 --- /dev/null +++ b/hosting/docker/clickhouse/users-override.xml @@ -0,0 +1,16 @@ + + + + 1 + 8192 + 1 + 0 + 0 + + 0 + 0 + 0 + 0 + + + diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index a8bc1167b77..b893c17eca9 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -177,6 +177,7 @@ services: - clickhouse:/var/lib/clickhouse - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro + - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro networks: - webapp healthcheck: From b55fba9e061420c0c5a12f7cedc037f085dba0b1 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:43:27 +0100 Subject: [PATCH 04/28] feat(run-store,run-engine): freeze the completed-waitpoints record and resolver contract (#4760) Builds on [#4754](https://github.com/triggerdotdev/trigger.dev/pull/4754), which added the store this contract belongs to. ## Why Two migrations are moving to Redis in parallel, and execution snapshots reference completed waitpoints across the boundary between them. If the record shape is agreed only once both halves are built, the correction lands mid-rollout: dual-write is live, real keys are in Redis, and changing the entry format then means two versions of the entry coexisting plus a migration for whatever was already written. Agreeing it now, while nothing writes a pointer, makes that same correction a type edit. The reserved-and-empty field is the same argument one level down. The entry format is what dual-write writes, so adding a field to it later splits the format in two. Reserving it before any write means the format never changes after writes begin. ## Summary Adds the type contract for carrying completed waitpoints alongside the Redis-backed execution-snapshot store: a `{cycleSeq, count}` pointer on the snapshot entry, the record shape that pointer resolves to, and the read-time resolver signature. Nothing constructs or reads a pointer yet, so this is inert on merge. The record shape has to reproduce `enhanceExecutionSnapshotWithWaitpoints` field for field, because that is what the executor consumes. A conformance test runs the real function against a reference resolver over an exhaustive grid of 6144 input combinations, derived from every `Waitpoint` column the function reads rather than hand-picked. ## Design `completedWaitpoints` is reserved on the entry type and always unset. `append()` rejects a set value, because the pointer's physical home is the `#c` sidecar field rather than the entry JSON. The append script mints both halves after the client serializes the entry, and the entry JSON has to stay byte-identical to the Postgres row so the two can be compared during a dual-write rollout. Two rules are worth calling out, both found by making the test fail rather than by reading the code: * `records` is the authoritative waitpoint set, not `order`. Only batch waits carry an index, so `order` is empty for a single `triggerAndWait` while the Postgres join still holds the id. Comparing id sets over `order` would serve the previous wait cycle's records. * `deriveFromRun` requires a non-null `completedByTaskRunId`. `Waitpoint.completedByTaskRun` is `onDelete: SetNull`, so an orphaned RUN waitpoint keeps its output with no run left to derive from. Those records carry their output inline instead. `tsconfig.freeze-test.json` typechecks the conformance test, which the package build config excludes. Without it, renaming a field in the frozen type compiles clean and every test stays green, so the literal assertions in the test would only pin the test's own writer. ## Fixes carried along Auditing the contract surfaced three defects in the append script, each with a regression test that fails when the fix is reverted: * A new wait cycle now clears any `records` left on a reused key. A `seq` counter lost to eviction can re-mint a `cycleSeq` whose key still holds another cycle's records, and `order` and `count` are overwritten together, so the mismatch check could not see the drift. * A carry-forward now attaches a pointer only if the current keyspace incarnation actually minted that cycle. The previous key-exists check adopted a dead incarnation's records under a count that agreed with them, reporting no mismatch. * The cycle-key size metric now counts `records`, not only `order`. It reported 7 bytes for a 20 KB key, so the high-water log could never fire on the field that grows. --- internal-packages/run-engine/package.json | 2 +- .../systems/completedWaitpointFreeze.test.ts | 664 ++++++++++++++++++ .../engine/systems/executionSnapshotSystem.ts | 2 +- .../run-engine/tsconfig.freeze-test.json | 19 + .../run-store/src/redisSnapshotStore.test.ts | 348 +++++++++ .../run-store/src/redisSnapshotStore.ts | 148 +++- 6 files changed, 1170 insertions(+), 13 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts create mode 100644 internal-packages/run-engine/tsconfig.freeze-test.json diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 96ace3a0e43..f4cfcd629d1 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -43,7 +43,7 @@ }, "scripts": { "clean": "rimraf dist", - "typecheck": "tsc --noEmit -p tsconfig.build.json", + "typecheck": "tsc --noEmit -p tsconfig.build.json && tsc --noEmit -p tsconfig.freeze-test.json", "test": "vitest --sequence.concurrent=false --no-file-parallelism", "test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled", "build": "pnpm run clean && tsc -p tsconfig.build.json", diff --git a/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts b/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts new file mode 100644 index 00000000000..a1af90582b6 --- /dev/null +++ b/internal-packages/run-engine/src/engine/systems/completedWaitpointFreeze.test.ts @@ -0,0 +1,664 @@ +// The freeze's executable definition. It runs the real oracle, +// enhanceExecutionSnapshotWithWaitpoints, against a reference resolver over equivalent +// records, and asserts the two agree field for field. The waitpoint lane owns the +// production resolver; this reference exists so the frozen shapes are checked rather +// than asserted. +import { isDeepStrictEqual } from "node:util"; +import { describe, expect, it } from "vitest"; +import type { Waitpoint } from "@trigger.dev/database"; +import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { CompletedWaitpoint } from "@trigger.dev/core/v3"; +import type { + CompletedWaitpointRecord, + CompletedWaitpointResolver, + CompletedWaitpointsPointer, + ResolveCompletedWaitpointsArgs, +} from "@internal/run-store"; + +// The frozen key sets, pinned exactly and bidirectionally. Renames, removals, widenings and +// required-to-optional all already break compilation through the usage sites below; an ADDED +// OPTIONAL field does not, and on a jointly-owned frozen type that is the change neither lane +// may make unilaterally. These fail on it. +type Exact = [A] extends [B] ? ([B] extends [A] ? true : never) : never; + +const _recordKeys: Exact< + keyof CompletedWaitpointRecord, + | "id" + | "friendlyId" + | "type" + | "completedAt" + | "outputType" + | "outputIsError" + | "output" + | "completedByTaskRunId" + | "completedByBatchId" + | "completedAfter" + | "idempotencyKey" +> = true; + +const _pointerKeys: Exact = true; + +const _argsKeys: Exact< + keyof ResolveCompletedWaitpointsArgs, + "runId" | "batchId" | "pointer" | "order" | "records" +> = true; +import { enhanceExecutionSnapshotWithWaitpoints } from "./executionSnapshotSystem.js"; + +function makeWaitpoint(overrides: Partial): Waitpoint { + return { + id: "wp_default", + friendlyId: "waitpoint_default", + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date("2026-01-01T00:00:00.000Z"), + idempotencyKey: "idem_generated", + userProvidedIdempotencyKey: false, + inactiveIdempotencyKey: null, + idempotencyKeyExpiresAt: null, + completedByTaskRunId: null, + completedByBatchId: null, + completedAfter: null, + output: null, + outputType: "application/json", + outputIsError: false, + projectId: "proj_1", + environmentId: "env_1", + tags: [], + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + ...overrides, + } as Waitpoint; +} + +// The WRITE side of the freeze: one Waitpoint row becomes one record. +function toRecord(w: Waitpoint): CompletedWaitpointRecord { + return { + id: w.id, + friendlyId: w.friendlyId, + type: w.type, + completedAt: (w.completedAt ?? new Date()).toISOString(), + outputType: w.outputType, + outputIsError: w.outputIsError, + output: recordOutputFor(w), + completedByTaskRunId: w.completedByTaskRunId ?? undefined, + completedByBatchId: w.completedByBatchId ?? undefined, + completedAfter: w.completedAfter?.toISOString(), + idempotencyKey: + w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey ? w.idempotencyKey : undefined, + }; +} + +function recordOutputFor(w: Waitpoint): CompletedWaitpointRecord["output"] { + if (w.output === null) return null; + // A RUN success re-derives byte-identically from TaskRun.output. A RUN error cannot, + // because TaskRun.error is jsonb, so it carries inline. + // + // This branch is deliberately BEFORE the application/store branch. An offloaded RUN + // success is still deriveFromRun, and that is correct: completeAttemptSuccess receives + // the same `output` and `outputType` the waitpoint got, so TaskRun.output holds the + // same ref string. The re-read stays byte-identical either way. + if (w.type === "RUN" && !w.outputIsError && w.completedByTaskRunId) + return { deriveFromRun: true }; + if (w.outputType === "application/store") return { ref: w.output }; + return { inline: w.output }; +} + +// The READ side of the freeze. Iterates `records`, never `order`. +async function referenceResolver( + args: ResolveCompletedWaitpointsArgs, + lookupRunOutput: (runId: string) => Promise +): Promise { + const out: CompletedWaitpoint[] = []; + for (const record of args.records) { + const indexes: (number | undefined)[] = []; + for (let i = 0; i < args.order.length; i++) { + if (args.order[i] === record.id) indexes.push(i); + } + if (indexes.length === 0) indexes.push(undefined); + + let output: string | undefined; + if (record.output === null) { + output = undefined; + } else if ("inline" in record.output) { + output = record.output.inline; + } else if ("ref" in record.output) { + output = record.output.ref; + } else if ("deriveFromRun" in record.output) { + output = record.completedByTaskRunId + ? await lookupRunOutput(record.completedByTaskRunId) + : undefined; + } else { + const _never: never = record.output; + throw new Error(`unknown record output variant: ${JSON.stringify(_never)}`); + } + + for (const index of indexes) { + out.push({ + id: record.id, + // Unreachable: the oracle's own loop pushes a non-negative integer or undefined. + // Reproduced because the frozen index-expansion rule names it. + index: index === -1 ? undefined : index, + friendlyId: record.friendlyId, + type: record.type, + completedAt: new Date(record.completedAt), + idempotencyKey: record.idempotencyKey, + completedByTaskRun: record.completedByTaskRunId + ? { + id: record.completedByTaskRunId, + friendlyId: RunId.toFriendlyId(record.completedByTaskRunId), + batch: args.batchId + ? { id: args.batchId, friendlyId: BatchId.toFriendlyId(args.batchId) } + : undefined, + } + : undefined, + completedAfter: record.completedAfter ? new Date(record.completedAfter) : undefined, + completedByBatch: record.completedByBatchId + ? { + id: record.completedByBatchId, + friendlyId: BatchId.toFriendlyId(record.completedByBatchId), + } + : undefined, + output, + outputType: record.outputType, + outputIsError: record.outputIsError, + }); + } + } + return out; +} + +// Proves the frozen hook signature is implementable exactly as declared. The reference +// resolver takes its TaskRun lookup as a second parameter, so the production shape is +// the curried form -- which is what the waitpoint lane will bind to a Prisma client. +// If this assignment stops compiling, the frozen signature has drifted. This file is +// typechecked by tsconfig.freeze-test.json, wired into this package's `typecheck` +// script, so that drift is caught -- vitest's esbuild transform alone would not catch it. +const resolverUnderTest: CompletedWaitpointResolver = (args) => + referenceResolver(args, async () => undefined); + +// The oracle spreads the snapshot, so it needs the two fields the mapping reads. +function makeSnapshot(batchId: string | null) { + return { id: "snap_1", runId: "run_1", batchId, checkpoint: null } as never; +} + +async function assertParity( + waitpoints: Waitpoint[], + order: string[], + batchId: string | null, + runOutputs: Record = {} +) { + const enhanced = enhanceExecutionSnapshotWithWaitpoints(makeSnapshot(batchId), waitpoints, order); + const args: ResolveCompletedWaitpointsArgs = { + runId: "run_1", + batchId: batchId ?? undefined, + pointer: { cycleSeq: 1, count: order.length }, + order, + records: waitpoints.map(toRecord), + }; + // count-carried-forward behaviour (order.length, not the record count) is covered by + // the run-store Redis suite, not here -- this line only constructs `args`, not asserts. + const resolved = await referenceResolver(args, async (id) => runOutputs[id]); + expect(resolved).toEqual(enhanced.completedWaitpoints); + return { enhanced, resolved }; +} + +describe("the frozen record shape", () => { + // Literals, not a mirror of the writer. A field rename or an encoding change must + // fail HERE, because the parity suite cannot see it. + it("pins the RUN record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_run", + friendlyId: "waitpoint_run", + type: "RUN", + completedByTaskRunId: "run_child", + output: '{"value":42}', + }) + ) + ).toEqual({ + id: "wp_run", + friendlyId: "waitpoint_run", + type: "RUN", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { deriveFromRun: true }, + completedByTaskRunId: "run_child", + completedByBatchId: undefined, + completedAfter: undefined, + idempotencyKey: undefined, + }); + }); + + it("pins the BATCH record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_batch", + friendlyId: "waitpoint_batch", + type: "BATCH", + completedByBatchId: "batch_child", + output: "Batch waitpoint completed", + }) + ) + ).toEqual({ + id: "wp_batch", + friendlyId: "waitpoint_batch", + type: "BATCH", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "Batch waitpoint completed" }, + completedByTaskRunId: undefined, + completedByBatchId: "batch_child", + completedAfter: undefined, + idempotencyKey: undefined, + }); + }); + + it("pins the DATETIME record", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_dt", + friendlyId: "waitpoint_dt", + type: "DATETIME", + completedAfter: new Date("2026-02-02T00:00:00.000Z"), + }) + ) + ).toEqual({ + id: "wp_dt", + friendlyId: "waitpoint_dt", + type: "DATETIME", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: null, + completedByTaskRunId: undefined, + completedByBatchId: undefined, + completedAfter: "2026-02-02T00:00:00.000Z", + idempotencyKey: undefined, + }); + }); + + it("pins the MANUAL record, with a user idempotency key and an offloaded output", () => { + expect( + toRecord( + makeWaitpoint({ + id: "wp_manual", + friendlyId: "waitpoint_manual", + type: "MANUAL", + idempotencyKey: "idem_user", + userProvidedIdempotencyKey: true, + output: "s3://bucket/key", + outputType: "application/store", + }) + ) + ).toEqual({ + id: "wp_manual", + friendlyId: "waitpoint_manual", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/store", + outputIsError: false, + output: { ref: "s3://bucket/key" }, + completedByTaskRunId: undefined, + completedByBatchId: undefined, + completedAfter: undefined, + idempotencyKey: "idem_user", + }); + }); + + it("keeps an offloaded RUN success on deriveFromRun, not ref", () => { + // Branch precedence. TaskRun.output holds the same ref string, so the re-read is + // still byte-identical. A later edit that reorders the branches must fail here. + const record = toRecord( + makeWaitpoint({ + id: "wp_run_offloaded", + type: "RUN", + completedByTaskRunId: "run_child", + output: "s3://bucket/key", + outputType: "application/store", + }) + ); + expect(record.output).toEqual({ deriveFromRun: true }); + }); + + it("carries a RUN error inline, never deriveFromRun", () => { + const record = toRecord( + makeWaitpoint({ + id: "wp_run_err", + type: "RUN", + completedByTaskRunId: "run_child", + output: '{"type":"BUILT_IN_ERROR"}', + outputIsError: true, + }) + ); + expect(record.output).toEqual({ inline: '{"type":"BUILT_IN_ERROR"}' }); + }); +}); + +// The pointer's shape is pinned by CompletedWaitpointsPointer and tsconfig.freeze-test.json, +// not by a runtime assertion here -- a value that only echoes its own construction can't fail. + +describe("the completed-waitpoints freeze", () => { + it("expands a repeated id at each of its positions", async () => { + const w = makeWaitpoint({ id: "wp_a", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], ["wp_a", "wp_other", "wp_a"], "batch_1"); + expect(resolved.map((r) => r.index)).toEqual([0, 2]); + }); + + it("yields one entry with an undefined index for a record absent from order", async () => { + const w = makeWaitpoint({ id: "wp_absent", type: "MANUAL" }); + const { resolved } = await assertParity([w], ["wp_other"], null); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.index).toBeUndefined(); + }); + + it("resolves a non-batch wait, where order is empty and one record exists", async () => { + // The commonest resume. Postgres's join holds the id while order does not, which is + // why `records` is authoritative and the mint comparison never reads `order`. + // batchId is null here on purpose: a triggerAndWait outside a batch is the shape + // this case is named for, and it exercises the oracle's `batchId ? ... : undefined` + // false branch, which no other case reaches. + // output stays null, so no TaskRun lookup is involved: both halves yield undefined. + const w = makeWaitpoint({ id: "wp_single", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], [], null); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.index).toBeUndefined(); + expect(resolved[0]!.completedByTaskRun?.id).toBe("run_child"); + expect(resolved[0]!.completedByTaskRun?.batch).toBeUndefined(); + }); + + it("keys completedByBatch on the id alone, not on the type", async () => { + // The oracle checks completedByBatchId without looking at `type`, at + // executionSnapshotSystem.ts:107-113. A resolver keyed on type would pass every + // other case here and diverge in production. + const w = makeWaitpoint({ + id: "wp_manual_with_batch", + type: "MANUAL", + completedByBatchId: "batch_child", + }); + const { resolved } = await assertParity([w], ["wp_manual_with_batch"], null); + expect(resolved[0]!.completedByBatch?.id).toBe("batch_child"); + }); + + it("carries outputIsError on a non-RUN type", async () => { + const w = makeWaitpoint({ + id: "wp_manual_err", + type: "MANUAL", + output: '{"type":"STRING_ERROR"}', + outputIsError: true, + }); + const { resolved } = await assertParity([w], ["wp_manual_err"], null); + expect(resolved[0]!.outputIsError).toBe(true); + expect(resolved[0]!.output).toBe('{"type":"STRING_ERROR"}'); + }); + + it("resolves through the frozen hook signature", async () => { + // Exercises resolverUnderTest, so the declared CompletedWaitpointResolver type is + // proved implementable at runtime, on top of the compile-time proof at its + // declaration above (checked by tsconfig.freeze-test.json). + const w = makeWaitpoint({ id: "wp_hook", type: "MANUAL" }); + const resolved = await resolverUnderTest({ + runId: "run_1", + batchId: undefined, + pointer: { cycleSeq: 1, count: 1 }, + order: ["wp_hook"], + records: [toRecord(w)], + }); + expect(resolved).toHaveLength(1); + expect(resolved[0]!.id).toBe("wp_hook"); + expect(resolved[0]!.index).toBe(0); + }); + + it("round-trips all four waitpoint types", async () => { + const waitpoints = [ + makeWaitpoint({ id: "wp_run", type: "RUN", completedByTaskRunId: "run_child" }), + makeWaitpoint({ id: "wp_batch", type: "BATCH", completedByBatchId: "batch_child" }), + makeWaitpoint({ + id: "wp_dt", + type: "DATETIME", + completedAfter: new Date("2026-02-02T00:00:00.000Z"), + }), + makeWaitpoint({ id: "wp_manual", type: "MANUAL" }), + ]; + const { resolved } = await assertParity(waitpoints, ["wp_run", "wp_batch"], "batch_1"); + expect(resolved.map((r) => r.type)).toEqual(["RUN", "BATCH", "DATETIME", "MANUAL"]); + }); + + it("applies the idempotency-key rule in all four combinations", async () => { + const combos: Array<[boolean, string | null, string | undefined]> = [ + [true, null, "idem_user"], + [true, "cleared", undefined], + [false, null, undefined], + [false, "cleared", undefined], + ]; + for (const [userProvided, inactive, expected] of combos) { + const w = makeWaitpoint({ + id: "wp_idem", + idempotencyKey: "idem_user", + userProvidedIdempotencyKey: userProvided, + inactiveIdempotencyKey: inactive, + }); + const { resolved } = await assertParity([w], ["wp_idem"], null); + expect(resolved[0]!.idempotencyKey).toBe(expected); + } + }); + + it("forwards completedAfter on a MANUAL waitpoint with a timeout", async () => { + // The plan comment scopes completedAfter to DATETIME. The oracle forwards it for any + // type, so the resolver must too. + const w = makeWaitpoint({ + id: "wp_timeout", + type: "MANUAL", + completedAfter: new Date("2026-03-03T00:00:00.000Z"), + }); + const { resolved } = await assertParity([w], ["wp_timeout"], null); + expect(resolved[0]!.completedAfter).toEqual(new Date("2026-03-03T00:00:00.000Z")); + }); + + it("falls back off deriveFromRun when the completing run was deleted", async () => { + const w = makeWaitpoint({ + id: "wp_orphan", + type: "RUN", + completedByTaskRunId: null, + output: '{"value":42}', + }); + const { resolved } = await assertParity([w], ["wp_orphan"], null); + expect(resolved[0]!.output).toBe('{"value":42}'); + }); + + it("maps every output variant", async () => { + const runSuccess = makeWaitpoint({ + id: "wp_run_ok", + type: "RUN", + completedByTaskRunId: "run_ok", + output: '{"value":42}', + }); + const runError = makeWaitpoint({ + id: "wp_run_err", + type: "RUN", + completedByTaskRunId: "run_err", + output: '{"type":"BUILT_IN_ERROR"}', + outputIsError: true, + }); + const offloaded = makeWaitpoint({ + id: "wp_ref", + type: "MANUAL", + output: "s3://bucket/key", + outputType: "application/store", + }); + const empty = makeWaitpoint({ id: "wp_none", type: "MANUAL", output: null }); + + const { resolved } = await assertParity( + [runSuccess, runError, offloaded, empty], + ["wp_run_ok", "wp_run_err", "wp_ref", "wp_none"], + "batch_1", + // deriveFromRun: the same string TaskRun.output holds verbatim. + { run_ok: '{"value":42}' } + ); + expect(resolved.map((r) => r.output)).toEqual([ + '{"value":42}', + '{"type":"BUILT_IN_ERROR"}', + "s3://bucket/key", + undefined, + ]); + }); +}); + +describe("the exhaustive parity grid", () => { + // Dimensions mirror every Waitpoint column the oracle reads (type, output, outputType, + // outputIsError, completedByTaskRunId, completedByBatchId, completedAfter, + // userProvidedIdempotencyKey, inactiveIdempotencyKey), plus order-membership and the + // reading entry's batchId. A new column the oracle reads must widen a dimension here, + // so the pinned combination count below fails instead of coverage silently shrinking. + const TYPES: Waitpoint["type"][] = ["RUN", "BATCH", "DATETIME", "MANUAL"]; + const OUTPUTS: (string | null)[] = [null, '{"value":42}']; + const OUTPUT_TYPES = ["application/json", "application/store"]; + const OUTPUT_IS_ERRORS = [false, true]; + const TASK_RUN_IDS: (string | null)[] = [null, "run_child"]; + const BATCH_IDS: (string | null)[] = [null, "batch_child"]; + const COMPLETED_AFTERS: (Date | null)[] = [null, new Date("2026-02-02T00:00:00.000Z")]; + const IDEMPOTENCY_COMBOS: Array<[boolean, string | null]> = [ + [false, null], + [false, "cleared"], + [true, null], + [true, "cleared"], + ]; + const ORDER_MEMBERSHIPS = ["absent", "once", "twice"] as const; + const READING_BATCH_IDS: (string | null)[] = [null, "batch_reading_entry"]; + + // Only reached when type is RUN, output is set, outputIsError is false, and + // completedByTaskRunId is "run_child": the deriveFromRun branch. The value matches + // OUTPUTS' non-null entry so a correct resolver is byte-identical to the oracle. + const RUN_OUTPUT_LOOKUP: Record = { run_child: '{"value":42}' }; + + it("agrees with the oracle across every combination", async () => { + type Combo = { + type: Waitpoint["type"]; + output: string | null; + outputType: string; + outputIsError: boolean; + completedByTaskRunId: string | null; + completedByBatchId: string | null; + completedAfter: Date | null; + userProvidedIdempotencyKey: boolean; + inactiveIdempotencyKey: string | null; + orderMembership: (typeof ORDER_MEMBERSHIPS)[number]; + readingBatchId: string | null; + }; + const failures: Array<{ combo: Combo; oracle: unknown; resolver: unknown }> = []; + let cases = 0; + + for (const type of TYPES) { + for (const output of OUTPUTS) { + for (const outputType of OUTPUT_TYPES) { + for (const outputIsError of OUTPUT_IS_ERRORS) { + for (const completedByTaskRunId of TASK_RUN_IDS) { + for (const completedByBatchId of BATCH_IDS) { + for (const completedAfter of COMPLETED_AFTERS) { + for (const [ + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + ] of IDEMPOTENCY_COMBOS) { + for (const orderMembership of ORDER_MEMBERSHIPS) { + for (const readingBatchId of READING_BATCH_IDS) { + cases++; + const combo: Combo = { + type, + output, + outputType, + outputIsError, + completedByTaskRunId, + completedByBatchId, + completedAfter, + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + orderMembership, + readingBatchId, + }; + + const id = "wp_grid"; + const w = makeWaitpoint({ + id, + type, + output, + outputType, + outputIsError, + completedByTaskRunId, + completedByBatchId, + completedAfter, + idempotencyKey: "idem_user", + userProvidedIdempotencyKey, + inactiveIdempotencyKey, + }); + const order = + orderMembership === "absent" + ? ["wp_other"] + : orderMembership === "once" + ? [id] + : [id, id]; + + const enhanced = enhanceExecutionSnapshotWithWaitpoints( + makeSnapshot(readingBatchId), + [w], + order + ); + const args: ResolveCompletedWaitpointsArgs = { + runId: "run_1", + batchId: readingBatchId ?? undefined, + pointer: { cycleSeq: 1, count: order.length }, + order, + records: [toRecord(w)], + }; + const resolved = await referenceResolver( + args, + async (runId) => RUN_OUTPUT_LOOKUP[runId] + ); + + if (!isDeepStrictEqual(resolved, enhanced.completedWaitpoints)) { + failures.push({ + combo, + oracle: enhanced.completedWaitpoints, + resolver: resolved, + }); + } + } + } + } + } + } + } + } + } + } + } + + expect(cases).toBe(6144); + expect( + failures.length, + failures.length > 0 + ? `${failures.length}/${cases} combinations diverged. First: ${JSON.stringify(failures[0], null, 2)}` + : undefined + ).toBe(0); + }); +}); + +describe("the freeze's two deliberate divergences", () => { + it("pins completedAt at write time, where the oracle samples the clock", () => { + // The oracle applies `w.completedAt ?? new Date()`, so a null value changes on every + // read. No deterministic record can match that. The record pins it once instead. + const w = makeWaitpoint({ id: "wp_null_at", completedAt: null }); + const record = toRecord(w); + expect(Math.abs(Date.now() - new Date(record.completedAt).getTime())).toBeLessThan(60_000); + }); + + it("takes batch{} from the reading entry, not the completing run's own batch", async () => { + // A known, deliberate conflation in the oracle. Byte-compatibility requires it. + const w = makeWaitpoint({ id: "wp_run", type: "RUN", completedByTaskRunId: "run_child" }); + const { resolved } = await assertParity([w], ["wp_run"], "batch_reading_entry"); + expect(resolved[0]!.completedByTaskRun?.batch?.id).toBe("batch_reading_entry"); + }); +}); diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index e79383a8bb6..81c41d2c2ae 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -58,7 +58,7 @@ function enhanceExecutionSnapshot( * Transforms a snapshot (with checkpoint but without waitpoints) into an EnhancedExecutionSnapshot * by combining it with pre-fetched waitpoints. */ -function enhanceExecutionSnapshotWithWaitpoints( +export function enhanceExecutionSnapshotWithWaitpoints( snapshot: ExecutionSnapshotWithCheckpoint, waitpoints: Waitpoint[], completedWaitpointOrder: string[] diff --git a/internal-packages/run-engine/tsconfig.freeze-test.json b/internal-packages/run-engine/tsconfig.freeze-test.json new file mode 100644 index 00000000000..284a132e293 --- /dev/null +++ b/internal-packages/run-engine/tsconfig.freeze-test.json @@ -0,0 +1,19 @@ +// Typechecks completedWaitpointFreeze.test.ts, which tsconfig.build.json otherwise excludes +// (src/**/*.test.ts) and vitest's esbuild transform never checks. This config has no +// "@triggerdotdev/source" customCondition, so it resolves @internal/run-store from its built +// `dist`, not from source -- same as tsconfig.build.json. That means this gate only sees a +// source change in run-store once run-store has been rebuilt, so it MUST be run through turbo +// (`pnpm run typecheck --filter @internal/run-engine`), whose `typecheck` task declares +// `dependsOn: ["^build"]`. Running `tsc -p tsconfig.freeze-test.json` (or `pnpm run typecheck`) +// directly inside this package, against a stale dist/, passes green while the frozen type has +// already drifted in source. Do not "fix" this with customConditions: that pulls +// @trigger.dev/core's source in too, which fails to typecheck here on `lib: ES2020`. +{ + "extends": "./tsconfig.build.json", + "include": ["src/engine/systems/completedWaitpointFreeze.test.ts"], + "exclude": [], + "compilerOptions": { + "composite": false, + "declaration": false + } +} diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts index 369d79e5338..0c7b4c0720a 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts @@ -10,6 +10,8 @@ import { isValidFor, RedisSnapshotStore, type SnapshotEntryInput, + type CompletedWaitpointsPointer, + type CompletedWaitpointRecord, } from "./redisSnapshotStore.js"; describe("snapshotKeys", () => { @@ -236,6 +238,145 @@ describe("append", () => { } ); + redisTest( + "round-trips a typed records array through the cycle hash's records field", + async ({ redisOptions }) => { + // The only place CompletedWaitpointRecord[] physically enters Redis. If the writer ever + // serializes a different envelope, this is where that would show up as a broken round trip. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 }); + const raw = createRedisClient(redisOptions); + const records: CompletedWaitpointRecord[] = [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "RUN", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { deriveFromRun: true }, + completedByTaskRunId: "run_child", + }, + ]; + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_a", index: 0 }], + records, + }, + }); + + const storedRaw = await raw.hget("snap:{run_1}:wp:1", "records"); + expect(JSON.parse(storedRaw!)).toEqual(records); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "a recordless new cycle clears another cycle's records off a reused key", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 }); + const raw = createRedisClient(redisOptions); + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_a", index: 0 }], + records: [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "stale" }, + }, + ], + }, + }); + expect(await raw.hget("snap:{run_1}:wp:1", "records")).not.toBeNull(); + + // Only the counter is lost, as under maxmemory eviction. A birth does not check seq, so + // the next new cycle re-mints cycleSeq 1 onto the surviving key. + await raw.del("snap:{run_1}:seq"); + + await store.append({ + entry: entry({ id: "snap_2" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] }, + }); + + expect(await raw.hget("snap:{run_1}:wp:1", "order")).toBe(JSON.stringify(["w_b"])); + expect(await raw.hget("snap:{run_1}:wp:1", "records")).toBeNull(); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "a carry-forward refuses a cycle this incarnation never minted", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + const raw = createRedisClient(redisOptions); + try { + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_old", index: 0 }], + records: [ + { + id: "w_old", + friendlyId: "waitpoint_old", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "stale" }, + }, + ], + }, + }); + + // Lose the whole keyspace except the cycle key, as under maxmemory eviction. + await raw.del("snap:{run_1}:e", "snap:{run_1}:idx", "snap:{run_1}:cur", "snap:{run_1}:seq"); + + const carried = await store.append({ + entry: entry({ id: "snap_2" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "carryForward", cycleSeq: 1 }, + }); + + // Written, flagged, and carrying NO pointer: the dead incarnation's waitpoints must not + // be served to a fresh run under a count that agrees with them. + expect(carried).toMatchObject({ outcome: "written", cycleMismatch: true }); + expect(carried).not.toHaveProperty("cycleSeq"); + const read = await store.getLatest("run_1"); + expect(read?.cycle).toBeUndefined(); + expect(read?.completedWaitpointIds).toBeUndefined(); + } finally { + raw.disconnect(); + await store.quit(); + } + } + ); + redisTest( "reports a duplicate id without overwriting the original entry", async ({ redisOptions }) => { @@ -384,6 +525,129 @@ describe("cycle keys", () => { }); }); +describe("cycle key deletion sweep", () => { + const KEY_SUFFIXES = ["e", "idx", "cur", "seq", "wp:1", "wp:2"] as const; + const INJECTION_POINTS = ["beforeCarryForward", "beforeSecondNewCycle"] as const; + + function powerset(items: readonly T[]): T[][] { + let out: T[][] = [[]]; + for (const item of items) { + out = out.concat(out.map((s) => [...s, item])); + } + return out; + } + + // Mechanized replacement for the two hand-picked eviction regressions above: replays the same + // birth/carryForward/new-cycle/terminal shape once per element of the powerset of key deletions, + // at two points in the sequence, and checks that a cycle key's records never leak into a pointer + // naming a different cycle than the one the key's own order field currently describes. + redisTest( + "records read back for a pointer never mention an id outside that pointer's own order", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + const raw = createRedisClient(redisOptions); + type Violation = { + subset: string[]; + injectionPoint: string; + runId: string; + pointer: CompletedWaitpointsPointer; + recordsFound: unknown; + }; + const violations: Violation[] = []; + let replays = 0; + + try { + for (const injectionPoint of INJECTION_POINTS) { + for (const subset of powerset(KEY_SUFFIXES)) { + replays++; + const runId = `run_sweep_${injectionPoint}_${replays}`; + const base = `snap:{${runId}}`; + const damage = async () => { + if (subset.length > 0) { + await raw.del(...subset.map((s) => `${base}:${s}`)); + } + }; + + await store.append({ + entry: entry({ id: "snap_1", runId }), + kind: "birth", + isTerminal: false, + cycle: { + kind: "new", + completedWaitpoints: [{ id: "w_c1", index: 0 }], + records: [ + { + id: "w_c1", + friendlyId: "waitpoint_c1", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "payload_c1" }, + }, + ], + }, + }); + + if (injectionPoint === "beforeCarryForward") await damage(); + await store.append({ + entry: entry({ id: "snap_2", runId }), + kind: "transition", + isTerminal: false, + cycle: { kind: "carryForward", cycleSeq: 1 }, + }); + + if (injectionPoint === "beforeSecondNewCycle") await damage(); + // birth, not transition: both eviction regressions above only reproduce past a birth's + // liveness bypass -- a transition here would just report skippedNoKeyspace once e or seq + // is gone, exempting the replay before the cycle-mint branch ever ran. + await store.append({ + entry: entry({ id: "snap_3", runId }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_c2", index: 0 }] }, + }); + + await store.append({ + entry: entry({ id: "snap_4", runId, executionStatus: "FINISHED" }), + kind: "transition", + isTerminal: true, + }); + + for (const id of ["snap_1", "snap_2", "snap_3", "snap_4"]) { + const read = await store.getById(runId, id); + if (!read?.cycle) continue; + const orderIds = new Set(read.completedWaitpointIds?.order ?? []); + const recordsRaw = await raw.hget(`${base}:wp:${read.cycle.cycleSeq}`, "records"); + if (recordsRaw === null) continue; + const recordIds = (JSON.parse(recordsRaw) as { id: string }[]).map((r) => r.id); + if (recordIds.some((rid) => !orderIds.has(rid))) { + violations.push({ + subset, + injectionPoint, + runId, + pointer: read.cycle, + recordsFound: recordIds, + }); + } + } + } + } + } finally { + raw.disconnect(); + await store.quit(); + } + + if (violations.length > 0) { + throw new Error( + `${violations.length} violation(s) across ${replays} replays. First: ` + + JSON.stringify(violations[0]) + ); + } + } + ); +}); + describe("read-side cycle mismatch", () => { redisTest( "warns and records a metric when a cycle's count disagrees with its order", @@ -1113,6 +1377,53 @@ describe("hash tag and keyPrefix", () => { }); describe("observability", () => { + redisTest("cycle-key bytes cover records, not just order", async ({ redisOptions }) => { + // The plan puts a metric on the wp: KEY size. records dominates that key once + // populated, so measuring order alone understates it by orders of magnitude. + const calls: Array<[string, number]> = []; + const metrics = { + recordAppend: () => {}, + recordEntryBytes: () => {}, + recordCycleKeyBytes: (b: number) => calls.push(["cycleBytes", b]), + recordCycleCount: () => {}, + recordSkippedNoKeyspace: () => {}, + recordCycleMismatch: () => {}, + recordLatency: () => {}, + }; + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000, metrics }); + try { + const records: CompletedWaitpointRecord[] = [ + { + id: "w_a", + friendlyId: "waitpoint_a", + type: "MANUAL", + completedAt: "2026-01-01T00:00:00.000Z", + outputType: "application/json", + outputIsError: false, + output: { inline: "y".repeat(20_000) }, + }, + ]; + const orderJson = JSON.stringify(["w_a"]); + const recordsJson = JSON.stringify(records); + + await store.append({ + entry: entry({ id: "snap_1" }), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }], records }, + }); + + expect(calls).toEqual([ + [ + "cycleBytes", + Buffer.byteLength(orderJson, "utf8") + Buffer.byteLength(recordsJson, "utf8"), + ], + ]); + } finally { + await store.quit(); + } + }); + redisTest("records sizes and outcomes without ever rejecting", async ({ redisOptions }) => { const calls: unknown[][] = []; const metrics = { @@ -1224,3 +1535,40 @@ describe("observability", () => { } ); }); + +describe("the reserved completedWaitpoints field", () => { + redisTest("append rejects an entry that sets it", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + try { + const runId = "run_reserved_throw"; + const pointer: CompletedWaitpointsPointer = { cycleSeq: 1, count: 0 }; + await expect( + store.append({ + entry: { ...entry({ id: "snap_1", runId }), completedWaitpoints: pointer }, + kind: "birth", + isTerminal: false, + }) + ).rejects.toThrow(/reserved/i); + } finally { + await store.quit(); + } + }); + + redisTest("a stored entry never holds the key", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + try { + const runId = "run_reserved_absent"; + await store.append({ + entry: entry({ id: "snap_1", runId }), + kind: "birth", + isTerminal: false, + }); + const read = await store.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.raw).not.toContain("completedWaitpoints"); + expect(read!.entry).not.toHaveProperty("completedWaitpoints"); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 7b60843e2b4..2964959c4fe 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -6,6 +6,7 @@ import { type Result, } from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; +import type { CompletedWaitpoint } from "@trigger.dev/core/v3/schemas"; export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string }; @@ -33,6 +34,88 @@ export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; } +// --------------------------------------------------------------------------- +// The completed-waitpoints freeze. Frozen jointly with the waitpoint lane. +// Do not change a field here without re-agreeing the contract with that lane. +// --------------------------------------------------------------------------- + +/** + * The once-per-wait-cycle pointer. `cycleSeq` names the snap:{runId}:wp: + * key. `count` is order.length -- NOT the record count -- so it is zero for any + * wait that carries no batch index. + */ +export type CompletedWaitpointsPointer = { + cycleSeq: number; + count: number; +}; + +/** + * A record's output. + * - `inline` holds the literal value. MANUAL and DATETIME are bounded by the offload + * thresholds; error outputs are not (only BUILT_IN_ERROR truncates), so the bound is + * the completion body limit. Postgres holds the same strings, so this is a copy. + * - `ref` holds an application/store reference that was already offloaded. + * - `deriveFromRun` means the resolver reads TaskRun.output for completedByTaskRunId. + * Only a RUN record with outputIsError false AND a non-null completedByTaskRunId uses + * it: TaskRun.output is a String column holding the same string verbatim, so the + * re-read is byte-identical. A RUN error cannot use it, because TaskRun.error is + * jsonb and never round-trips. Waitpoint.completedByTaskRun is onDelete: SetNull, so + * an orphaned RUN waitpoint (the completing run row was deleted) has no run left to + * derive from -- its output carries inline instead. + */ +export type CompletedWaitpointRecordOutput = + | { inline: string } + | { ref: string } + | { deriveFromRun: true } + | null; + +/** + * One completed waitpoint, one per DISTINCT id in a wait cycle. The resolver expands + * this into one CompletedWaitpoint per position of the id in the cycle's order list. + */ +export type CompletedWaitpointRecord = { + id: string; + friendlyId: string; + type: "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + /** ISO. The writer pins it, applying the null fallback once. */ + completedAt: string; + /** Defaults to "application/json" at source. */ + outputType: string; + outputIsError: boolean; + output: CompletedWaitpointRecordOutput; + /** RUN. The resolver derives friendlyId, and batch{} from the READING entry's batchId. */ + completedByTaskRunId?: string; + /** BATCH. The resolver derives friendlyId. */ + completedByBatchId?: string; + /** ISO. Any type may set it: a MANUAL waitpoint with a timeout does. */ + completedAfter?: string; + /** Already resolved: userProvidedIdempotencyKey && !inactiveIdempotencyKey. */ + idempotencyKey?: string; +}; + +/** + * What the store hands the resolver. The store owns the keyspace, so the store reads + * and parses the cycle hash. The resolver never touches Redis and never derives a key. + */ +export type ResolveCompletedWaitpointsArgs = { + runId: string; + /** The batchId of the entry being READ, never the entry that minted the cycle. */ + batchId?: string; + pointer: CompletedWaitpointsPointer; + /** Index oracle only. A SUBSET of the record ids. Repeats preserved. */ + order: string[]; + /** The authoritative, complete set. Iterate this, never `order`. */ + records: CompletedWaitpointRecord[]; +}; + +/** + * This lane owns the signature. The waitpoint lane owns the implementation, which + * lives in run-engine because a deriveFromRun record needs a Postgres read. + */ +export type CompletedWaitpointResolver = ( + args: ResolveCompletedWaitpointsArgs +) => Promise; + export type SnapshotEntryInput = { id: string; engine: "V2"; @@ -53,6 +136,16 @@ export type SnapshotEntryInput = { runnerId?: string; metadata?: unknown; error?: string; + /** + * RESERVED. Always unset. `append` rejects a set value. + * + * The pointer's physical form is the `#c` sidecar field on the `e` hash, + * because the append Lua mints both halves after the client serializes the entry. + * The entry JSON must stay byte-identical to the caller's document, and the Postgres + * snapshot row has no pointer column, so a pointer inside the JSON would stop the two + * documents from being comparable for the dual-write comparator. + */ + completedWaitpoints?: CompletedWaitpointsPointer; }; export type WaitpointIds = { present: boolean; distinctIds: string[]; order: string[] }; @@ -67,7 +160,7 @@ export type SnapshotRead = { isValid: boolean; entry: Record; raw: string; - cycle?: { cycleSeq: number; count: number }; + cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; }; @@ -155,9 +248,20 @@ export class RedisSnapshotStore { isTerminal: boolean; expectedCur?: string; cycle?: - | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[]; records?: string } + | { + kind: "new"; + completedWaitpoints: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + } | { kind: "carryForward"; cycleSeq: number }; }): Promise { + if (args.entry.completedWaitpoints !== undefined) { + throw new Error( + "completedWaitpoints is a reserved entry field and must stay unset. The pointer's " + + "physical form is the `#c` sidecar field, which the append script mints. " + + "Writing it into the entry JSON breaks byte-comparability with the Postgres row." + ); + } return this.#timed("append", async () => { const k = snapshotKeys(args.entry.runId); const raw = JSON.stringify(args.entry); @@ -172,7 +276,7 @@ export class RedisSnapshotStore { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); - records = args.cycle.records ?? ""; + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; @@ -199,11 +303,17 @@ export class RedisSnapshotStore { args.expectedCur !== undefined ? "1" : "0" )) as string[]; - return this.#interpretAppend(reply, raw, orderJson, args.entry.runId); + return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); }); } - #interpretAppend(reply: string[], raw: string, orderJson: string, runId: string): AppendResult { + #interpretAppend( + reply: string[], + raw: string, + orderJson: string, + records: string, + runId: string + ): AppendResult { if (reply[0] === SKIPPED) { this.metrics?.recordSkippedNoKeyspace(); this.metrics?.recordAppend("skippedNoKeyspace", "none"); @@ -224,7 +334,7 @@ export class RedisSnapshotStore { if (cycleMismatch) { this.metrics?.recordCycleMismatch(); } - this.#observeSizes(raw, orderJson, cycleSeq, runId); + this.#observeSizes(raw, orderJson, records, cycleSeq, runId); this.metrics?.recordAppend("written", ttl); return { outcome: "written", @@ -235,14 +345,21 @@ export class RedisSnapshotStore { }; } - #observeSizes(raw: string, orderJson: string, cycleSeq: number, runId: string): void { + #observeSizes( + raw: string, + orderJson: string, + records: string, + cycleSeq: number, + runId: string + ): void { const entryBytes = Buffer.byteLength(raw, "utf8"); this.metrics?.recordEntryBytes(entryBytes); if (this.highWater.entryBytes !== undefined && entryBytes > this.highWater.entryBytes) { this.logger.warn("RedisSnapshotStore entry above high-water mark", { runId, entryBytes }); } if (orderJson !== "") { - const cycleBytes = Buffer.byteLength(orderJson, "utf8"); + // The whole wp: key, not just its order field: records dominates it once populated. + const cycleBytes = Buffer.byteLength(orderJson, "utf8") + Buffer.byteLength(records, "utf8"); this.metrics?.recordCycleKeyBytes(cycleBytes); if (this.highWater.cycleKeyBytes !== undefined && cycleBytes > this.highWater.cycleKeyBytes) { this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { @@ -469,13 +586,22 @@ export class RedisSnapshotStore { redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) if records ~= '' then redis.call('HSET', wpKey(cycleSeq), 'records', records) + else + -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key + -- still holds another cycle's records, and order/count stay mutually consistent so the + -- mismatch check cannot see it. No-op on a fresh key. + redis.call('HDEL', wpKey(cycleSeq), 'records') end elseif cycleMode == 'carry' then - cycleSeq = cycleSeqIn - local c = redis.call('HGET', wpKey(cycleSeq), 'count') - if not c then + -- Attach a pointer only if this incarnation actually minted the cycle. seq can be + -- evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- incarnation's order and records under a consistent count, invisibly. + local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') + local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') + if not c or minted < cycleSeqIn then mismatch = 1 else + cycleSeq = cycleSeqIn orderCount = c end end From f98e303292bde1eec7ce3ad89fa831e065eafed8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:23:58 +0100 Subject: [PATCH 05/28] feat(webapp): resolve which shard an environment mints run roots into (#4755) ## Summary Adds the shard-selection stage of run-id minting. `resolveMintShard(env)` returns which run-ops database an environment mints its new run roots into: the active shard list, then a fleet-wide override, then a per-environment or per-organization pin, then a rendezvous hash of the environment id. That half is inert. Nothing calls `resolveMintShard`, no deployment has any of the new flags set, and an empty active list returns the current answer without reading anything. **The other half is not inert, and it is where review effort belongs.** To stamp a grace window this needs a read-then-write under a lock, so it rewrites the global feature-flag write path that `runOpsMintKind` already depends on in production. See below. ## Placement Resolution reads the active list from a global flag, applies the grace window, and then picks: - a fleet-wide override if one is set, which is how a cutover completes without visiting each organization. `new` holds the whole fleet on the current id format. - otherwise a per-environment or per-organization pin. `new` holds one organization back while the rest move, which is how a canary works. - otherwise a rendezvous hash, so adding a shard moves only about 1/(N+1) of environments and removing one moves only its own. Two hash details are load-bearing. Scores are 64-bit `sha256(envId \0 key)`, because a 32-bit score collides at our environment count and an undetected tie would resolve by iteration order. The parsed key list is sorted, because otherwise two deployments listing the same shards in a different CSV order would place environments differently. A pin or override naming a shard that has left the active list falls through to the hash and reports once. Honouring it would leak the drain the active list exists to perform, and throwing would fail triggers whenever a pinned shard drains. ## Why the active list is a flag and not an environment variable A deploy rolls for hours, so two pods hold two different environment values at the same time. A list held in the environment therefore splits the fleet for the length of the rollout, with new pods placing an environment on one shard and old pods on another. A grace window measured in seconds cannot cover that, and the same knob times the existing mint-kind flip so it cannot simply be lengthened. An environment variable also cannot record its own flip time, and an operator cannot know a rollout's end in advance. So the list, its grace stamp and the override are global flags, written server-side against the control-plane clock under an advisory lock. This branch adds no environment variables. ## The write path, which is live Stamping generalises to any number of graced flag groups in one transaction under one lock. That has three consequences a reviewer should look at directly: - It closes a real bug. `runOpsMintKind` is an editable control on the global flags page, and that page previously wrote it with a bare upsert: no lock, no stamp. An operator flipping mint kind through the UI got an ungraced flip, so every pod crossed the cutover at a different moment. Verified against a running instance, before and after. - A graced group is all-or-nothing. Submitting its primary writes the group with a fresh stamp; omitting it deletes the primary and its stamp together, because a stamp left without its primary keeps being served and would mint into a shard just removed. - The advisory lock takes the previous id as well as the current one, in a fixed order, so writers on an older release still serialise during a rollout. The legacy id can be dropped one release after this ships. This folds with #4751 rather than replacing it: its `unlockLockedFlags` rule decides what the sweep may delete, and the graced groups keep their stamp under the lock. Both sets of tests pass. ## Notes for review Determinism is a property of the pure core for fixed inputs. The wrapper supplies the clock, the same split `effectiveMintKind` already uses. A failed read of the list falls back to the current id format rather than guessing. Six flags appear in the admin pages immediately. The two pins are per-organization, so they render read-only on the global page. The list, its stamp and the override are deployment-wide, so they render read-only in the organization dialog. Nothing bounds the active list against shards that actually exist. That is safe while nothing mints, but the change that carries a shard key into an id must land after the shard descriptors bound the list, or bound it itself. --- .../app/components/admin/flagChangeList.ts | 56 +++ .../app/routes/admin.api.v1.feature-flags.ts | 28 +- .../webapp/app/routes/admin.feature-flags.tsx | 55 +- apps/webapp/app/v3/featureFlags.server.ts | 242 ++++++--- apps/webapp/app/v3/featureFlags.ts | 114 ++++- .../mintShardAssignment.test.ts | 475 ++++++++++++++++++ .../v3/runOpsMigration/mintShardAssignment.ts | 206 ++++++++ .../v3/runOpsMigration/mintShardGrace.test.ts | 234 +++++++++ .../app/v3/runOpsMigration/mintShardGrace.ts | 133 +++++ .../runOpsMigration/runOpsMintShard.server.ts | 92 ++++ .../test/adminFeatureFlagsRouteAction.test.ts | 20 +- apps/webapp/test/globalFlagChangeList.test.ts | 142 ++++++ .../test/globalFlagWriteRouting.test.ts | 110 ++++ .../test/runOpsMintGlobalFlipLock.test.ts | 8 +- apps/webapp/test/runOpsMintShardFlags.test.ts | 118 +++++ .../test/runOpsMintShardSetFlip.test.ts | 279 ++++++++++ knip.json | 3 +- 17 files changed, 2192 insertions(+), 123 deletions(-) create mode 100644 apps/webapp/app/components/admin/flagChangeList.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts create mode 100644 apps/webapp/test/globalFlagChangeList.test.ts create mode 100644 apps/webapp/test/globalFlagWriteRouting.test.ts create mode 100644 apps/webapp/test/runOpsMintShardFlags.test.ts create mode 100644 apps/webapp/test/runOpsMintShardSetFlip.test.ts diff --git a/apps/webapp/app/components/admin/flagChangeList.ts b/apps/webapp/app/components/admin/flagChangeList.ts new file mode 100644 index 00000000000..512d3758bc7 --- /dev/null +++ b/apps/webapp/app/components/admin/flagChangeList.ts @@ -0,0 +1,56 @@ +import { derivedFlagsClearedWith } from "~/v3/featureFlags"; + +export type FlagChange = + | { key: string; type: "added"; newVal: string } + | { key: string; type: "removed"; oldVal: string } + | { key: string; type: "changed"; oldVal: string; newVal: string }; + +/** + * What a global flag save will do, for the confirm dialog. + * + * A graced primary that is unset also clears its stamps. Those keys are locked, so the caller + * filters them out of `initialValues` — the cascade therefore reads `storedValues`, which is the + * unfiltered set the loader returned. Reading `initialValues` finds nothing and understates the + * deletion, which is the defect this parameter exists to prevent. + */ +export function buildFlagChangeList(params: { + editableKeys: readonly string[]; + lockedKeys: readonly string[]; + initialValues: Record; + storedValues: Record; + newValues: Record; +}): FlagChange[] { + const { editableKeys, initialValues, storedValues, newValues } = params; + + return editableKeys.flatMap((key) => { + const wasSet = key in initialValues; + const isSet = key in newValues; + const oldVal = initialValues[key]; + const newVal = newValues[key]; + + if (!wasSet && !isSet) return []; + if (wasSet && isSet && stableValue(oldVal) === stableValue(newVal)) return []; + + if (!wasSet && isSet) { + return [{ key, type: "added", newVal: String(newVal) }]; + } + + if (wasSet && !isSet) { + // Only an unset clears the stamps. A change re-stamps instead. + const cascaded = derivedFlagsClearedWith(key) + .filter((derived) => derived in storedValues) + .map((derived) => ({ + key: derived, + type: "removed", + oldVal: String(storedValues[derived]), + })); + return [{ key, type: "removed", oldVal: String(oldVal) }, ...cascaded]; + } + + return [{ key, type: "changed", oldVal: String(oldVal), newVal: String(newVal) }]; + }); +} + +function stableValue(value: unknown): string { + return JSON.stringify(value ?? null); +} diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index 8cd4f77873e..e9da02effd9 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -3,7 +3,12 @@ import { json } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; -import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { + applyGlobalGracedFlips, + makeSetMultipleFlags, + touchesGracedGroup, + withoutDerivedKeys, +} from "~/v3/featureFlags.server"; import { validatePartialFeatureFlags } from "~/v3/featureFlags"; export async function action({ request }: ActionFunctionArgs) { @@ -25,19 +30,16 @@ export async function action({ request }: ActionFunctionArgs) { ); } - // Derived grace-stamp fields are computed server-side; never trust them from the body. - const { - runOpsMintKindPrev: _ignoredPrev, - runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags - } = validationResult.data; + // Both the strip and the branch derive from the graced-group table, so adding a group needs + // no edit here. Naming the keys inline is how a new group ends up writing its stamp straight + // from the request body, with no lock. + const requestedFlags = withoutDerivedKeys(validationResult.data) as Partial< + typeof validationResult.data + >; - // A global mint-kind flip stamps its grace window under a lock (applyGlobalMintKindFlip); - // any other flag save writes directly. - const updatedFlags = - requestedFlags.runOpsMintKind !== undefined - ? await applyGlobalMintKindFlip(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) - : await makeSetMultipleFlags(prisma)(requestedFlags); + const updatedFlags = touchesGracedGroup(requestedFlags) + ? await applyGlobalGracedFlips(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) + : await makeSetMultipleFlags(prisma)(requestedFlags); return json({ success: true, diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index 197800c7ef3..e987812f520 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -14,6 +14,7 @@ import { type FeatureFlagKey, type FlagControlType, getAllFlagControlTypes, + lockedFlagsInPayload, validatePartialFeatureFlags, } from "~/v3/featureFlags"; import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; @@ -29,6 +30,7 @@ import { DialogFooter, } from "~/components/primitives/Dialog"; import { cn } from "~/utils/cn"; +import { buildFlagChangeList } from "~/components/admin/flagChangeList"; import { UNSET_VALUE, BooleanControl, @@ -111,17 +113,12 @@ export const action = dashboardAction( const { isManagedCloud } = featuresForRequest(request); - // On managed cloud, reject if payload includes locked flags - if (isManagedCloud) { - const lockedInPayload = Object.keys(parsed.data.flags).filter((key) => - GLOBAL_LOCKED_FLAGS.includes(key) + const lockedInPayload = lockedFlagsInPayload(Object.keys(parsed.data.flags), isManagedCloud); + if (lockedInPayload.length > 0) { + return json( + { error: `Cannot modify locked flags: ${lockedInPayload.join(", ")}` }, + { status: 400 } ); - if (lockedInPayload.length > 0) { - return json( - { error: `Cannot modify locked flags: ${lockedInPayload.join(", ")}` }, - { status: 400 } - ); - } } const validationResult = validatePartialFeatureFlags(parsed.data.flags); @@ -137,6 +134,7 @@ export const action = dashboardAction( catalogKeys: Object.keys(getAllFlagControlTypes()) as FeatureFlagKey[], isManagedCloud, unlockLockedFlags: parsed.data.unlockLockedFlags ?? false, + graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, }); return json({ success: true }); @@ -401,6 +399,7 @@ export default function AdminFeatureFlagsRoute() { open={confirmOpen} onOpenChange={setConfirmOpen} initialValues={initialValues} + storedValues={allFlags} newValues={values} controlTypes={typedControlTypes} lockedKeys={unlocked ? [] : GLOBAL_LOCKED_FLAGS} @@ -467,6 +466,7 @@ function ConfirmDialog({ open, onOpenChange, initialValues, + storedValues, newValues, controlTypes, lockedKeys, @@ -477,6 +477,7 @@ function ConfirmDialog({ open: boolean; onOpenChange: (open: boolean) => void; initialValues: Record; + storedValues: Record; newValues: Record; controlTypes: Record; lockedKeys: readonly string[]; @@ -488,34 +489,12 @@ function ConfirmDialog({ .filter((key) => !lockedKeys.includes(key)) .sort(); - type Change = - | { key: string; type: "added"; newVal: string } - | { key: string; type: "removed"; oldVal: string } - | { key: string; type: "changed"; oldVal: string; newVal: string }; - - const changes = editableKeys.flatMap((key) => { - const wasSet = key in initialValues; - const isSet = key in newValues; - const oldVal = initialValues[key]; - const newVal = newValues[key]; - - if (!wasSet && !isSet) return []; - if (wasSet && isSet && stableStringify(oldVal) === stableStringify(newVal)) return []; - - if (!wasSet && isSet) { - return [{ key, type: "added" as const, newVal: String(newVal) }]; - } - if (wasSet && !isSet) { - return [{ key, type: "removed" as const, oldVal: String(oldVal) }]; - } - return [ - { - key, - type: "changed" as const, - oldVal: String(oldVal), - newVal: String(newVal), - }, - ]; + const changes = buildFlagChangeList({ + editableKeys, + lockedKeys, + initialValues, + storedValues, + newValues, }); return ( diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index dd1fb125ba6..152a14b6496 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -1,16 +1,18 @@ import { type z } from "zod"; import type { PrismaClient } from "@trigger.dev/database"; -import { boundedIn, prisma, type PrismaClientOrTransaction } from "~/db.server"; import { FEATURE_FLAG, type FeatureFlagCatalogSchema, type FeatureFlagKey, FeatureFlagCatalog, GLOBAL_LOCKED_FLAGS, + GRACED_FLAG_GROUPS, validatePartialFeatureFlags, } from "~/v3/featureFlags"; import { env } from "~/env.server"; import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; +import { stampMintShardSetFlip } from "~/v3/runOpsMigration/mintShardGrace"; +import { $transaction, boundedIn, prisma, type PrismaClientOrTransaction } from "~/db.server"; export type FlagsOptions = { key: T; @@ -182,56 +184,126 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma }; } -// Read -> stamp -> write the global mint-kind grace metadata in one transaction. The three -// FeatureFlag rows may not exist yet, so a row FOR UPDATE can't lock them; an advisory xact lock -// serializes concurrent global flips so one can't clobber another's grace stamp (mirrors per-org). -export async function applyGlobalMintKindFlip( - client: PrismaClient, - requestedFlags: Partial>, - graceMs: number -): Promise<{ key: string; value: any }[]> { - return client.$transaction(async (tx) => { - await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`; +// The key topology lives in the shared module, because the admin page needs it too. This adds +// the stamping behaviour, which is server-only. +const GRACED_GLOBAL_GROUPS = GRACED_FLAG_GROUPS.map((group) => ({ + ...group, + stamp: group.primary === FEATURE_FLAG.runOpsMintKind ? stampMintKindFlip : stampMintShardSetFlip, +})); - const existingRows = await tx.featureFlag.findMany({ - where: { - key: { - in: [ - FEATURE_FLAG.runOpsMintKind, - FEATURE_FLAG.runOpsMintKindPrev, - FEATURE_FLAG.runOpsMintKindFlippedAt, - ], - }, - }, - select: { key: true, value: true }, - }); - const existingGlobal: Record = {}; - for (const row of existingRows) { - existingGlobal[row.key] = row.value; +const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => [ + g.primary, + ...g.derived, +]); + +function gracedGroupFor(key: FeatureFlagKey) { + return GRACED_GLOBAL_GROUPS.find((g) => g.primary === key || g.derived.includes(key)); +} + +// True when a save changes any graced group, and therefore needs the stamped path. Derived from +// the group table, so adding a group cannot leave a caller silently writing an unstamped flip. +export function touchesGracedGroup(requestedFlags: Record): boolean { + return GRACED_GLOBAL_GROUPS.some((group) => requestedFlags[group.primary] !== undefined); +} + +// Strips every derived key: a grace stamp is computed here, never accepted from a caller. +// Only the flags whose stored value differs. Each write is a round trip inside an interactive +// transaction, so writing an unchanged flag costs a round trip for nothing. +export function flagsNeedingWrite( + requested: Record, + existing: Record +): Record { + const out: Record = {}; + for (const [key, value] of Object.entries(requested)) { + if (JSON.stringify(existing[key] ?? null) !== JSON.stringify(value ?? null)) { + out[key] = value; } + } + return out; +} - // Anchor the cutover to the control-plane DB clock, not this process's wall clock. - const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; +export function withoutDerivedKeys( + requestedFlags: Partial> +): Record { + const out: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + for (const derived of group.derived) { + delete out[derived]; + } + } + return out; +} - const stamped = stampMintKindFlip( - existingGlobal, - { ...requestedFlags }, - now.getTime(), - graceMs - ) as Partial>; +// The rows may not exist yet, so a row FOR UPDATE cannot lock them; an advisory xact lock +// serializes concurrent global flips instead, so one cannot clobber another's stamp. +// +// Two lock ids are taken, in a fixed order. The FIRST is the operative one: an older release +// takes only that id, and a deploy rolls for hours, so it is the id that serializes across both +// versions. The second is this release's name and adds nothing until every writer takes it. +// Renaming without keeping the old id is what would leave the two versions unserialized. Remove +// the legacy id one release after this one ships, when nothing takes it alone. +async function lockGracedGroups(tx: PrismaClientOrTransaction): Promise { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`; + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`; +} - return makeSetMultipleFlags(tx)(stamped); +// Reads each group's current rows and returns the requested flags plus a fresh stamp for every +// group the save actually changes. A group whose primary the save omits is left untouched. +async function stampGracedGroups( + tx: PrismaClientOrTransaction, + requestedFlags: Record, + graceMs: number +): Promise> { + const existingRows = await tx.featureFlag.findMany({ + where: { key: { in: boundedIn(GRACED_GLOBAL_KEYS) } }, + select: { key: true, value: true }, }); + const existingGlobal: Record = {}; + for (const row of existingRows) { + existingGlobal[row.key] = row.value; + } + + // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling + // deploy spans hours, so every pod must date the window against one shared clock. + const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; + + let stamped: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs); + } + return stamped; } -/** - * Replace-semantics write for the global admin flags page: catalog keys present in - * `requestedFlags` are upserted, catalog keys absent from it are deleted. - * - * A locked flag absent from the payload means the page never offered it for editing, not that - * the admin unset it, so it survives the sweep. Only a self-hosted page that says it unlocked - * them can delete one. - */ +// Merge-semantics write: sets what the caller asked for, stamps any graced group it changes, and +// touches nothing else. Used by the JSON admin API. +export async function applyGlobalGracedFlips( + client: PrismaClient, + requestedFlags: Partial>, + graceMs: number +): Promise<{ key: string; value: any }[]> { + const applied = await $transaction(client, "applyGlobalGracedFlips", async (tx) => { + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, withoutDerivedKeys(requestedFlags), graceMs); + return makeSetMultipleFlags(tx)(stamped as Partial>); + }); + + // The helper resolves undefined rather than throwing when Prisma swallows an infrastructure + // error. This write stamps a cutover window, so a transaction that did not run must be loud. + if (!applied) { + throw new Error("applyGlobalGracedFlips: transaction did not complete"); + } + return applied; +} + +// Replace-semantics write for the global admin flags page: submitted flags upsert, omitted ones +// delete unless protected. One transaction covers the stamp, the upserts and the deletes, so a +// save cannot half-apply. +// +// A graced group is all-or-nothing. Submitting its primary writes the group with a fresh stamp. +// Omitting its primary deletes the primary AND its stamp together, because a stamp left behind +// without its primary keeps being served: {set: [], prevSet: [a], flippedAt: t} resolves to [a] +// for the rest of the window, which would mint into a shard the operator just removed. The +// delete ignores `isProtected` for the derived keys for the same reason. export async function replaceGlobalFeatureFlags( client: PrismaClient, params: { @@ -239,33 +311,75 @@ export async function replaceGlobalFeatureFlags( catalogKeys: FeatureFlagKey[]; isManagedCloud: boolean; unlockLockedFlags: boolean; + graceMs: number; } ): Promise { + const requestedFlags = withoutDerivedKeys(params.requestedFlags); + + // A locked flag absent from the payload means the page never offered it, not that the admin + // unset it, so it survives. Only a self-hosted page that says it unlocked them may delete one. const canDeleteLocked = params.unlockLockedFlags && !params.isManagedCloud; - const upsertOps: ReturnType[] = []; - const keysToDelete: string[] = []; - - for (const key of params.catalogKeys) { - if (key in params.requestedFlags) { - const value = params.requestedFlags[key]; - upsertOps.push( - client.featureFlag.upsert({ - where: { key }, - create: { key, value: value as any }, - update: { value: value as any }, - }) + const isProtected = (key: FeatureFlagKey) => + !canDeleteLocked && GLOBAL_LOCKED_FLAGS.includes(key); + + const applied = await $transaction(client, "replaceGlobalFeatureFlags", async (tx) => { + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, requestedFlags, params.graceMs); + + const toWrite: Record = {}; + const keysToDelete: string[] = []; + + for (const key of params.catalogKeys) { + const group = gracedGroupFor(key); + + if (group) { + if (requestedFlags[group.primary] !== undefined) { + if (stamped[key] !== undefined) { + toWrite[key] = stamped[key]; + } + } else if (!isProtected(group.primary)) { + keysToDelete.push(key); + } + continue; + } + + if (key in requestedFlags) { + toWrite[key] = requestedFlags[key]; + } else if (!isProtected(key)) { + keysToDelete.push(key); + } + } + + // One round trip to learn the stored values, then a write only for what actually differs. + // makeSetMultipleFlags upserts sequentially, so an unchanged flag costs a round trip for + // nothing, and this transaction is interactive and holds a pooled connection. + const writeKeys = Object.keys(toWrite); + if (writeKeys.length > 0) { + const storedRows = await tx.featureFlag.findMany({ + where: { key: { in: boundedIn(writeKeys) } }, + select: { key: true, value: true }, + }); + const stored: Record = {}; + for (const row of storedRows) { + stored[row.key] = row.value; + } + + await makeSetMultipleFlags(tx)( + flagsNeedingWrite(toWrite, stored) as Partial> ); - } else if (canDeleteLocked || !GLOBAL_LOCKED_FLAGS.includes(key)) { - keysToDelete.push(key); } - } - await client.$transaction([ - ...upsertOps, - ...(keysToDelete.length > 0 - ? [client.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })] - : []), - ]); + if (keysToDelete.length > 0) { + await tx.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } }); + } + + return true; + }); + + // This write deletes flags, so a transaction that did not run must reach the caller. + if (!applied) { + throw new Error("replaceGlobalFeatureFlags: transaction did not complete"); + } } /** The global flag set, with the env-var defaults this app applies. */ diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 7c775799178..3a88beb54bc 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -26,6 +26,16 @@ export const FEATURE_FLAG = { // Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts. runOpsMintKindPrev: "runOpsMintKindPrev", runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt", + // Gen-2 mint shard pins, read from the org override blob only. See runOpsMintShard.server.ts. + runOpsMintShard: "runOpsMintShard", + runOpsMintShardEnvPins: "runOpsMintShardEnvPins", + // The active mint-shard list, global only. Lives here rather than in the environment because a + // rolling deploy runs two environment values at once for hours. See mintShardGrace.ts. + runOpsMintShardSet: "runOpsMintShardSet", + runOpsMintShardSetPrev: "runOpsMintShardSetPrev", + runOpsMintShardSetFlippedAt: "runOpsMintShardSetFlippedAt", + // Fleet-wide pin for the complete cutover. Beats every per-org and per-env pin. + runOpsMintShardOverride: "runOpsMintShardOverride", queueMetricsUiEnabled: "queueMetricsUiEnabled", // Per-organization rollout for creating additional environment API keys. additionalApiKeysEnabled: "additionalApiKeysEnabled", @@ -89,6 +99,52 @@ export const FeatureFlagCatalog = { // by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS). [FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]), [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(), + // Pins one org to a gen-2 mint shard. "new" holds the org on gen-1 run-ops ids, which is how + // a canary keeps the fleet's default while one org moves. Only honored while the key is in + // the active list; a drained key falls through to the hash. + [FEATURE_FLAG.runOpsMintShard]: z + .string() + .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'), + // Per-environment pins as JSON: {"": ""}. A JSON string because + // this catalog is scalar-only. Rejected at write, so a typo cannot silently un-pin an env. + [FEATURE_FLAG.runOpsMintShardEnvPins]: z.string().superRefine((raw, ctx) => { + const fail = (message: string) => ctx.addIssue({ code: z.ZodIssueCode.custom, message }); + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return fail("must be valid JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return fail("must be a JSON object mapping environment id to shard key"); + } + for (const [environmentId, value] of Object.entries(parsed)) { + if (typeof value !== "string" || !(value === "new" || /^[a-z0-9]$/.test(value))) { + fail(`"${environmentId}" must map to a single [a-z0-9] char, or "new"`); + } + } + }), + // CSV of the shard keys eligible for root minting right now. Empty means no gen-2 minting. + // Reserved keys are rejected, because "new" already means gen-1. + [FEATURE_FLAG.runOpsMintShardSet]: z.string().refine( + (v) => + v + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + .every((k) => /^[a-z0-9]$/.test(k)), + "must be a CSV of single [a-z0-9] chars" + ), + // Grace stamp: the previously-effective list and the flip time, written by + // stampMintShardSetFlip on a genuine change. Display-only (see ORG_LOCKED_FLAGS). + [FEATURE_FLAG.runOpsMintShardSetPrev]: z.string(), + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: z.string().datetime(), + // Sends every environment to one shard, outranking every pin, so a cutover needs no per-org + // visit. "new" holds the whole fleet on gen-1. Only honored while the key is in the active set. + [FEATURE_FLAG.runOpsMintShardOverride]: z + .string() + .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'), // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and // separate). Off unless enabled for the org. [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(), @@ -101,11 +157,20 @@ export const FeatureFlagCatalog = { export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; -// Infrastructure flags that are read-only on the global flags page. -// Shown with current/resolved value but no controls. +// Infrastructure flags, plus org-scoped-only flags, that are read-only on the global flags +// page. Shown with current/resolved value but no controls. An org-scoped-only flag belongs +// here because its resolver never reads a global row, so an editable global control would +// offer a setting that does nothing. export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.defaultWorkerInstanceGroupId, FEATURE_FLAG.taskEventRepository, + FEATURE_FLAG.runOpsMintShard, + FEATURE_FLAG.runOpsMintShardEnvPins, + // Grace stamps are computed server-side. An editable control here would discard what it saves. + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, ]; // Flags that are read-only on the org-level dialog. @@ -118,8 +183,53 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ // System-wide only — orgs must not be able to override these kill switches. FEATURE_FLAG.additionalApiKeyIssuanceEnabled, FEATURE_FLAG.additionalApiKeyLookupEnabled, + // The active mint-shard list is deployment-wide; only the pins are per-org. + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, +]; + +/** + * Flag groups where the operator sets a `primary` and the server computes the rest. The topology + * lives here, not in the server module, because the admin page needs it too: unsetting a primary + * clears its stamps, and the page has to disclose that. + */ +export const GRACED_FLAG_GROUPS: ReadonlyArray<{ + primary: FeatureFlagKey; + derived: readonly FeatureFlagKey[]; +}> = [ + { + primary: FEATURE_FLAG.runOpsMintKind, + derived: [FEATURE_FLAG.runOpsMintKindPrev, FEATURE_FLAG.runOpsMintKindFlippedAt], + }, + { + primary: FEATURE_FLAG.runOpsMintShardSet, + derived: [FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt], + }, ]; +/** The stamps deleted alongside `primary`. Empty unless `primary` is a graced primary. */ +export function derivedFlagsClearedWith(primary: string): FeatureFlagKey[] { + const group = GRACED_FLAG_GROUPS.find((g) => g.primary === primary); + return group ? [...group.derived] : []; +} + +/** + * Locked flags present in a payload the global page must refuse. On managed cloud the page never + * offers them, so their presence means the request did not come from that page. Locally an admin + * may unlock and edit them, so nothing is refused. + */ +export function lockedFlagsInPayload( + payloadKeys: string[], + isManagedCloud: boolean +): FeatureFlagKey[] { + if (!isManagedCloud) return []; + return payloadKeys.filter((key): key is FeatureFlagKey => + GLOBAL_LOCKED_FLAGS.includes(key as FeatureFlagKey) + ); +} + // Create a Zod schema from the existing catalog export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); export type FeatureFlagCatalog = z.infer; diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts new file mode 100644 index 00000000000..d88e64e1d75 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -0,0 +1,475 @@ +import { describe, expect, it } from "vitest"; +import { + computeMintShard, + resolveMintShardWith, + type MintShardCache, + type MintShardDeps, + type ResolveMintShardDeps, +} from "./mintShardAssignment"; +import { type MintShardSetResolution } from "./mintShardGrace"; + +const GRACE_MS = 90_000; +const T = 1_000_000; + +// Cuid-shaped ids, not sequential integers: a sequential space does not model the real +// key distribution the hash has to spread. +function envIds(count: number): string[] { + const ids: string[] = []; + for (let i = 0; i < count; i++) { + ids.push(`cm${(i * 2654435761).toString(36).padStart(10, "0")}${i.toString(36)}zzq`); + } + return ids; +} + +function deps( + resolution: MintShardSetResolution, + overrides: Partial = {} +): MintShardDeps { + return { + resolution, + nowMs: T + GRACE_MS + 1, + graceMs: GRACE_MS, + orgFeatureFlags: undefined, + ...overrides, + }; +} + +function orgFlags(flags: Record) { + return { orgFeatureFlags: flags }; +} + +function place(ids: string[], resolution: MintShardSetResolution): Map { + const out = new Map(); + for (const id of ids) { + out.set(id, computeMintShard({ id }, deps(resolution))); + } + return out; +} + +describe("computeMintShard — the no-shards answer", () => { + it("returns new when the live list is empty", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: [] }))).toBe("new"); + }); + + it("returns new when a stale stamp is present but both lists are empty", () => { + const resolution: MintShardSetResolution = { set: [], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); + + it("returns new when the grace serves an empty list", () => { + const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); + + it("returns new when the grace serves an empty prevSet", () => { + const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); +}); + +describe("computeMintShard — determinism", () => { + it("returns the same value for the same environment on every call", () => { + const resolution: MintShardSetResolution = { set: ["a", "b", "c"] }; + const first = computeMintShard({ id: "env_stable" }, deps(resolution)); + for (let i = 0; i < 1000; i++) { + expect(computeMintShard({ id: "env_stable" }, deps(resolution))).toBe(first); + } + }); + + it("ignores the order the operator listed the keys in", () => { + const ids = envIds(200); + const canonical = place(ids, { set: ["a", "b", "c"] }); + for (const permutation of [ + ["c", "b", "a"], + ["b", "a", "c"], + ["a", "c", "b"], + ]) { + expect(place(ids, { set: permutation })).toEqual(canonical); + } + }); +}); + +describe("computeMintShard — pins", () => { + const resolution: MintShardSetResolution = { set: ["a", "b"] }; + + it("lets a per-env pin override the hash", () => { + const ids = envIds(50); + for (const id of ids) { + const pinned = computeMintShard( + { id }, + deps(resolution, orgFlags({ runOpsMintShardEnvPins: JSON.stringify({ [id]: "b" }) })) + ); + expect(pinned).toBe("b"); + } + }); + + it("lets a per-org pin override the hash when no per-env pin is set", () => { + const ids = envIds(50); + for (const id of ids) { + expect(computeMintShard({ id }, deps(resolution, orgFlags({ runOpsMintShard: "a" })))).toBe( + "a" + ); + } + }); + + it("lets a per-env pin beat a per-org pin", () => { + const result = computeMintShard( + { id: "env_1" }, + deps( + resolution, + orgFlags({ + runOpsMintShard: "a", + runOpsMintShardEnvPins: JSON.stringify({ env_1: "b" }), + }) + ) + ); + expect(result).toBe("b"); + }); + + it("holds an environment on gen-1 when the pin is new", () => { + expect( + computeMintShard({ id: "env_1" }, deps(resolution, orgFlags({ runOpsMintShard: "new" }))) + ).toBe("new"); + expect( + computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShardEnvPins: JSON.stringify({ env_1: "new" }) })) + ) + ).toBe("new"); + }); + + it("falls through to the hash and reports when the pin is outside the active set", () => { + // Honouring a drained pin would leak the drain; throwing would fail customer triggers. + const rejected: string[] = []; + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, { + ...orgFlags({ runOpsMintShard: "z" }), + onPinRejected: (info) => rejected.push(info.pin), + }) + ); + expect(result).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + expect(rejected).toEqual(["z"]); + }); + + it("honours a pin to a drained key for the whole grace window, then falls through", () => { + const draining: MintShardSetResolution = { set: ["a"], prevSet: ["a", "b"], flippedAtMs: T }; + const pinnedToB = orgFlags({ runOpsMintShard: "b" }); + expect(computeMintShard({ id: "env_1" }, deps(draining, { ...pinnedToB, nowMs: T + 1 }))).toBe( + "b" + ); + expect( + computeMintShard({ id: "env_1" }, deps(draining, { ...pinnedToB, nowMs: T + GRACE_MS })) + ).not.toBe("b"); + }); + + it("ignores an unparseable pin blob rather than un-pinning silently", () => { + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShard: "a", runOpsMintShardEnvPins: "{not json" })) + ); + expect(result).toBe("a"); + }); + + it("falls back to the org pin when the blob holds an invalid value for this env", () => { + const result = computeMintShard( + { id: "env_1" }, + deps( + resolution, + orgFlags({ + runOpsMintShard: "a", + runOpsMintShardEnvPins: JSON.stringify({ env_1: "LEGACY" }), + }) + ) + ); + expect(result).toBe("a"); + }); + + it("ignores an invalid org pin value", () => { + const result = computeMintShard( + { id: "env_1" }, + deps(resolution, orgFlags({ runOpsMintShard: "legacy" })) + ); + expect(result).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + }); +}); + +describe("computeMintShard — rendezvous properties", () => { + const ids = envIds(10_000); + + it("spreads roughly evenly across the active set", () => { + for (const set of [ + ["a", "b"], + ["a", "b", "c"], + ["a", "b", "c", "d"], + ]) { + const counts = new Map(); + for (const shard of place(ids, { set }).values()) { + counts.set(shard, (counts.get(shard) ?? 0) + 1); + } + expect(counts.size).toBe(set.length); + const expected = ids.length / set.length; + for (const count of counts.values()) { + expect(Math.abs(count - expected) / expected).toBeLessThan(0.1); + } + } + }); + + it("moves about 1/(N+1) of environments when a shard is added", () => { + const cases: Array<{ from: string[]; to: string[]; expected: number }> = [ + { from: ["a"], to: ["a", "b"], expected: 1 / 2 }, + { from: ["a", "b"], to: ["a", "b", "c"], expected: 1 / 3 }, + { from: ["a", "b", "c"], to: ["a", "b", "c", "d"], expected: 1 / 4 }, + ]; + + for (const { from, to, expected } of cases) { + const before = place(ids, { set: from }); + const after = place(ids, { set: to }); + const added = to.filter((k) => !from.includes(k)); + let moved = 0; + for (const id of ids) { + if (before.get(id) === after.get(id)) continue; + moved++; + // HRW's defining property: a mover lands on the ADDED shard, never on a survivor. + expect(added).toContain(after.get(id)); + } + expect(Math.abs(moved / ids.length - expected) / expected).toBeLessThan(0.1); + } + }); + + it("moves only the environments that hashed to a removed shard", () => { + const before = place(ids, { set: ["a", "b", "c"] }); + const after = place(ids, { set: ["a", "b"] }); + for (const id of ids) { + if (before.get(id) === "c") { + expect(after.get(id)).not.toBe("c"); + } else { + expect(after.get(id)).toBe(before.get(id)); + } + } + }); + + it("also moves pinned environments when their shard is removed", () => { + // Criterion 6 is a property of the hash only. A pin to a removed key moves too. + const pinnedToC = orgFlags({ runOpsMintShard: "c" }); + expect(computeMintShard({ id: "env_1" }, deps({ set: ["a", "b", "c"] }, pinnedToC))).toBe("c"); + expect(computeMintShard({ id: "env_1" }, deps({ set: ["a", "b"] }, pinnedToC))).not.toBe("c"); + }); +}); + +describe("resolveMintShardWith — cache, read failure and fail-safe", () => { + function wrapperDeps( + overrides: Partial = {} + ): ResolveMintShardDeps & { reads: number } { + const state = { + readFlags: async () => ({ runOpsMintShardSet: "a,b" }), + cache: { current: undefined as MintShardCache }, + nowMs: T, + ttlMs: 30_000, + graceMs: GRACE_MS, + orgFeatureFlags: undefined as unknown, + reads: 0, + ...overrides, + }; + const wrapped = state.readFlags; + state.readFlags = async () => { + state.reads++; + return wrapped(); + }; + return state; + } + + it("reads once, then serves the cache until the TTL expires", async () => { + const deps = wrapperDeps(); + await resolveMintShardWith({ id: "env_1" }, deps); + await resolveMintShardWith({ id: "env_2" }, deps); + await resolveMintShardWith({ id: "env_3" }, deps); + expect(deps.reads).toBe(1); + }); + + it("reads again once the TTL expires", async () => { + const deps = wrapperDeps(); + await resolveMintShardWith({ id: "env_1" }, deps); + deps.nowMs = T + 30_000; + await resolveMintShardWith({ id: "env_1" }, deps); + expect(deps.reads).toBe(2); + }); + + it("falls back to gen-1 when the read throws, and does not poison the cache", async () => { + // A blip must not move every environment's placement, so it returns gen-1 rather than guess. + let fail = true; + const deps = wrapperDeps({ + readFlags: async () => { + if (fail) throw new Error("db down"); + return { runOpsMintShardSet: "a,b" }; + }, + }); + const failures: unknown[] = []; + deps.onReadFailed = (error) => failures.push(error); + + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + expect(failures).toHaveLength(1); + + fail = false; + expect(["a", "b"]).toContain(await resolveMintShardWith({ id: "env_1" }, deps)); + }); + + it("returns gen-1 when the stored list is empty", async () => { + const deps = wrapperDeps({ readFlags: async () => ({ runOpsMintShardSet: "" }) }); + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + }); + + it("coalesces concurrent misses into ONE read", async () => { + // Two misses must share a single read. Otherwise a slower read landing after a faster one + // writes its older snapshot back into the cache for a whole TTL. + let release: (flags: Record) => void = () => {}; + const gate = new Promise>((resolve) => { + release = resolve; + }); + const deps = wrapperDeps({ readFlags: () => gate }); + + const both = Promise.all([ + resolveMintShardWith({ id: "env_1" }, deps), + resolveMintShardWith({ id: "env_2" }, deps), + ]); + release({ runOpsMintShardSet: "a,b" }); + await both; + + expect(deps.reads).toBe(1); + }); + + it("does not let a slower read overwrite a newer one", async () => { + // The slow read starts first and finishes last. Its result must not become the cached + // value, because the fast read already published a newer snapshot. + let releaseSlow: (flags: Record) => void = () => {}; + const slow = new Promise>((resolve) => { + releaseSlow = resolve; + }); + let call = 0; + const deps = wrapperDeps({ + readFlags: () => { + call++; + return call === 1 ? slow : Promise.resolve({ runOpsMintShardSet: "c" }); + }, + }); + + const first = resolveMintShardWith({ id: "env_1" }, deps); + const second = resolveMintShardWith({ id: "env_2" }, deps); + releaseSlow({ runOpsMintShardSet: "a" }); + await Promise.all([first, second]); + + // One read served both, so there is no second snapshot to race with. + expect(deps.reads).toBe(1); + expect(deps.cache.current?.value.resolution.set).toEqual(["a"]); + }); + + it("clears the in-flight refresh after a failure, so the next call retries", async () => { + let fail = true; + const deps = wrapperDeps({ + readFlags: async () => { + if (fail) throw new Error("db down"); + return { runOpsMintShardSet: "a,b" }; + }, + }); + deps.onReadFailed = () => {}; + + expect(await resolveMintShardWith({ id: "env_1" }, deps)).toBe("new"); + fail = false; + expect(["a", "b"]).toContain(await resolveMintShardWith({ id: "env_1" }, deps)); + expect(deps.reads).toBe(2); + }); + + it("agrees with the pure core for the same inputs", async () => { + const deps = wrapperDeps(); + const viaWrapper = await resolveMintShardWith({ id: "env_1" }, deps); + const viaCore = computeMintShard( + { id: "env_1" }, + { + resolution: { set: ["a", "b"] }, + nowMs: T, + graceMs: GRACE_MS, + orgFeatureFlags: undefined, + } + ); + expect(viaWrapper).toBe(viaCore); + }); +}); + +describe("computeMintShard — the global override wins the complete cutover", () => { + const resolution: MintShardSetResolution = { set: ["a", "b"] }; + + it("beats the hash for every environment", () => { + for (const id of envIds(200)) { + expect(computeMintShard({ id }, deps(resolution, { globalOverride: "b" }))).toBe("b"); + } + }); + + it("beats a per-org pin", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { globalOverride: "b", orgFeatureFlags: { runOpsMintShard: "a" } }) + ); + expect(shard).toBe("b"); + }); + + it("beats a per-env pin, which is the whole point of a cutover", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { + globalOverride: "b", + orgFeatureFlags: { runOpsMintShardEnvPins: JSON.stringify({ env_1: "a" }) }, + }) + ); + expect(shard).toBe("b"); + }); + + it("holds the whole fleet on gen-1 when set to new, whatever any org pinned", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { globalOverride: "new", orgFeatureFlags: { runOpsMintShard: "a" } }) + ); + expect(shard).toBe("new"); + }); + + it("is ignored, and reported, when it names a key outside the active set", () => { + // Honouring it would mint into a drained or unroutable shard. Explicit pins still apply. + const rejected: string[] = []; + const shard = computeMintShard( + { id: "env_1" }, + deps(resolution, { + globalOverride: "z", + orgFeatureFlags: { runOpsMintShard: "a" }, + onOverrideRejected: (info) => rejected.push(info.override), + }) + ); + expect(shard).toBe("a"); + expect(rejected).toEqual(["z"]); + }); + + it("reports a bad override WITHOUT the environment id, so one line covers the fleet", () => { + // Keying the report by environment would log once per environment for a fleet-wide setting. + const seen: Array<{ override: string }> = []; + for (const id of envIds(50)) { + computeMintShard( + { id }, + deps(resolution, { globalOverride: "z", onOverrideRejected: (i) => seen.push(i) }) + ); + } + expect(seen).toHaveLength(50); + expect(new Set(seen.map((i) => i.override))).toEqual(new Set(["z"])); + expect(seen.every((i) => !("environmentId" in i))).toBe(true); + }); + + it("is ignored when it is not a legal value", () => { + for (const bad of ["legacy", "AB", "", "a,b"]) { + const shard = computeMintShard({ id: "env_1" }, deps(resolution, { globalOverride: bad })); + expect(shard).toBe(computeMintShard({ id: "env_1" }, deps(resolution))); + } + }); + + it("cannot resurrect minting when the list is empty", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: [] }, { globalOverride: "b" }))).toBe( + "new" + ); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts new file mode 100644 index 00000000000..a49a1a6a60d --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts @@ -0,0 +1,206 @@ +// PURE module: no env, no clock, no database. Kept separate from the .server wrapper so a test +// can drive it without evaluating env.server, whose schema parse demands a full environment. +import { createHash } from "node:crypto"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { + effectiveMintShardSet, + GEN_1_PIN_VALUE, + isValidPinValue, + readMintShardSetResolution, + type MintShardSetResolution, +} from "./mintShardGrace"; + +export type MintShardDeps = { + // The live list, from the control-plane database. + resolution: MintShardSetResolution; + // Fleet-wide pin that beats every per-org and per-env pin. The complete-cutover lever. + globalOverride?: unknown; + nowMs: number; + graceMs: number; + orgFeatureFlags: unknown; + onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; +}; + +function asRecord(value: unknown): Record | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return value as Record; +} + +// Map keys are environment INTERNAL ids (cuids), not friendly ids. An unparseable blob, or a +// blob whose value for this environment is invalid, yields no per-env pin and lets the +// per-org scalar decide — never a silent un-pin straight to the hash. +function readEnvPin(raw: unknown, environmentId: string): ShardKey | undefined { + if (typeof raw !== "string") return undefined; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + + const pins = asRecord(parsed); + const pin = pins?.[environmentId]; + return isValidPinValue(pin) ? pin : undefined; +} + +// Both pins live in the org override blob the trigger path already holds, so resolving a mint +// shard costs no query. +function readPin(orgFeatureFlags: unknown, environmentId: string): ShardKey | undefined { + const blob = asRecord(orgFeatureFlags); + if (!blob) return undefined; + + const envPin = readEnvPin(blob[FEATURE_FLAG.runOpsMintShardEnvPins], environmentId); + if (envPin !== undefined) return envPin; + + const scalar = blob[FEATURE_FLAG.runOpsMintShard]; + return isValidPinValue(scalar) ? scalar : undefined; +} + +// 64 bits: a 32-bit score collides at this system's environment count, and an undetected tie +// would resolve by iteration order. The NUL separates the fields so no two input pairs can +// concatenate alike. This hash input is FROZEN once gen-2 minting is live: changing it +// re-places every environment, silently. +function shardScore(environmentId: string, key: string): bigint { + return createHash("sha256").update(`${environmentId}\0${key}`).digest().readBigUInt64BE(0); +} + +function hrwSelect(environmentId: string, activeSet: string[]): string { + let bestKey = activeSet[0]; + let bestScore = shardScore(environmentId, bestKey); + + for (let i = 1; i < activeSet.length; i++) { + const key = activeSet[i]; + const score = shardScore(environmentId, key); + if (score > bestScore || (score === bestScore && key > bestKey)) { + bestKey = key; + bestScore = score; + } + } + + return bestKey; +} + +// PURE CORE — no env, no clock, no I/O; tests drive this directly. Deterministic for fixed +// deps, which is what lets run minting and token minting agree on one answer. +// +// An empty list is the off state, and it is the state of every deployment that has not set the +// flag. Bounding the list against the shard keys this deployment can actually route belongs with +// the shard descriptors, which own that information; nothing here mints, so nothing can misroute. +// +// A pin outside the active set falls through to the hash rather than throwing: honouring it +// would leak the drain the active list performs, and throwing would fail customer triggers +// whenever a pinned shard drains. +export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey { + const activeSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + if (activeSet.length === 0) { + return "new"; + } + + // The global override outranks every pin, so one flag completes a cutover without visiting + // each org. An override outside the active set is ignored, so explicit pins still apply. + if (isValidPinValue(deps.globalOverride)) { + const override = deps.globalOverride; + if (override === GEN_1_PIN_VALUE) { + return "new"; + } + if (activeSet.includes(override)) { + return override; + } + // Fleet-wide, so it is reported once for the value, not once per environment. + deps.onOverrideRejected?.({ override, activeSet }); + } + + const pin = readPin(deps.orgFeatureFlags, environment.id); + if (pin !== undefined) { + if (pin === GEN_1_PIN_VALUE) { + return "new"; + } + if (activeSet.includes(pin)) { + return pin; + } + deps.onPinRejected?.({ environmentId: environment.id, pin, activeSet }); + } + + return hrwSelect(environment.id, activeSet); +} + +// Read together so the override costs no extra query beyond the list it is bounded by. + +type GlobalShardConfig = { resolution: MintShardSetResolution; override: unknown }; + +export type MintShardCache = { value: GlobalShardConfig; expiresAt: number } | undefined; + +type MintShardCacheHandle = { + current: MintShardCache; + // The refresh currently in flight, if any. Concurrent misses share it. + inFlight?: Promise; +}; + +export type ResolveMintShardDeps = { + // Reads the list rows. Injected so the cache and the fail-safe are testable without a + // database, the same way computeRunIdMintKind takes its flag reader. + readFlags: () => Promise>; + cache: MintShardCacheHandle; + nowMs: number; + ttlMs: number; + graceMs: number; + orgFeatureFlags: unknown; + onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; + onReadFailed?: (error: unknown) => void; +}; + +// The live list is org-independent, so one process-wide entry serves every mint: one query per +// process per TTL, over one round-trip. Two processes can therefore disagree for the TTL PLUS the +// replica lag behind the read, which can exceed graceMs. That is tolerable here and only here, +// because a gen-2 id carries its own shard key, so disagreement cannot misroute an existing run; +// it only decides where the next root lands, and every failure direction is toward gen-1. +// +// A failed read falls back to gen-1 rather than guessing a list. Guessing would move every +// environment's placement for the length of one blip. +async function refreshConfig(deps: ResolveMintShardDeps): Promise { + try { + const flags = await deps.readFlags(); + const config: GlobalShardConfig = { + resolution: readMintShardSetResolution(flags), + override: flags[FEATURE_FLAG.runOpsMintShardOverride], + }; + deps.cache.current = { value: config, expiresAt: deps.nowMs + deps.ttlMs }; + return config; + } finally { + deps.cache.inFlight = undefined; + } +} + +export async function resolveMintShardWith( + environment: { id: string; orgFeatureFlags?: unknown }, + deps: ResolveMintShardDeps +): Promise { + let config: GlobalShardConfig; + const cached = deps.cache.current; + if (cached && cached.expiresAt > deps.nowMs) { + config = cached.value; + } else { + try { + // Single-flight. Without it, two misses both read, and a slower read landing after a + // faster one puts its older snapshot back into the cache for a whole TTL. + config = await (deps.cache.inFlight ??= refreshConfig(deps)); + } catch (error) { + deps.onReadFailed?.(error); + return "new"; + } + } + + return computeMintShard(environment, { + resolution: config.resolution, + globalOverride: config.override, + nowMs: deps.nowMs, + graceMs: deps.graceMs, + orgFeatureFlags: deps.orgFeatureFlags, + onPinRejected: deps.onPinRejected, + onOverrideRejected: deps.onOverrideRejected, + }); +} diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts new file mode 100644 index 00000000000..3d26cb9fd1a --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it } from "vitest"; +import { generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; +import { + effectiveMintShardSet, + isValidPinValue, + parseShardCsv, + readMintShardSetResolution, + SHARD_KEY_PATTERN, + stampMintShardSetFlip, + type MintShardSetResolution, +} from "./mintShardGrace"; + +const GRACE_MS = 90_000; +const T = 1_000_000; + +describe("parseShardCsv", () => { + it("returns an empty list for unset, empty and whitespace input", () => { + expect(parseShardCsv(undefined)).toEqual([]); + expect(parseShardCsv("")).toEqual([]); + expect(parseShardCsv(" ")).toEqual([]); + expect(parseShardCsv(",,")).toEqual([]); + }); + + it("trims, dedupes and SORTS, so operator typing order cannot change HRW", () => { + expect(parseShardCsv("b, a ,b")).toEqual(["a", "b"]); + expect(parseShardCsv("a,b,c")).toEqual(parseShardCsv("c,b,a")); + expect(parseShardCsv("b,c,a")).toEqual(parseShardCsv("a,c,b")); + }); + + it("accepts every one of the 36 legal shard keys", () => { + const all = "abcdefghijklmnopqrstuvwxyz0123456789".split(""); + expect(parseShardCsv(all.join(","))).toEqual([...all].sort()); + }); + + it("throws on a key outside [a-z0-9]", () => { + // generateRunOpsIdV2 throws on these; an unvalidated key MUST fail at boot, not at mint. + expect(() => parseShardCsv("A")).toThrow(/shard key/i); + expect(() => parseShardCsv("ab")).toThrow(/shard key/i); + expect(() => parseShardCsv("a,-")).toThrow(/shard key/i); + expect(() => parseShardCsv("a,_")).toThrow(/shard key/i); + }); + + it("rejects the reserved keys by name", () => { + expect(() => parseShardCsv("new")).toThrow(/reserved/i); + expect(() => parseShardCsv("a,legacy")).toThrow(/reserved/i); + }); +}); + +// Core does not export its shard-char pattern, so pin the local one to the real minter. +describe("shard alphabet agrees with the core minter", () => { + it("accepts exactly the characters generateRunOpsIdV2 accepts", () => { + const candidates = [ + ..."abcdefghijklmnopqrstuvwxyz0123456789".split(""), + ..."ABZ-_. +/é!".split(""), + "", + "ab", + ]; + + for (const candidate of candidates) { + let minterAccepts = true; + try { + generateRunOpsIdV2(candidate); + } catch { + minterAccepts = false; + } + + expect(SHARD_KEY_PATTERN.test(candidate)).toBe(minterAccepts); + } + }); +}); + +describe("isValidPinValue", () => { + it('accepts a shard key, and accepts "new" as the gen-1 hold value', () => { + expect(isValidPinValue("a")).toBe(true); + expect(isValidPinValue("7")).toBe(true); + expect(isValidPinValue("new")).toBe(true); + }); + + it("rejects legacy, and rejects anything outside the alphabet", () => { + expect(isValidPinValue("legacy")).toBe(false); + expect(isValidPinValue("A")).toBe(false); + expect(isValidPinValue("ab")).toBe(false); + expect(isValidPinValue("")).toBe(false); + }); +}); + +describe("effectiveMintShardSet", () => { + it("returns set when there is no stamp", () => { + const r: MintShardSetResolution = { set: ["a", "b"] }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("returns set when flippedAtMs is absent even though prevSet is present", () => { + const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"] }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("serves prevSet inside the window and set at/after the boundary", () => { + const r: MintShardSetResolution = { set: ["a", "b"], prevSet: ["a"], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual(["a"]); + expect(effectiveMintShardSet(r, T + GRACE_MS - 1, GRACE_MS)).toEqual(["a"]); + // Boundary is exclusive on the prev side, so every process crosses it together. + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a", "b"]); + expect(effectiveMintShardSet(r, T + GRACE_MS + 1, GRACE_MS)).toEqual(["a", "b"]); + }); + + it("represents a graced first activation as an empty prevSet", () => { + const r: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T, GRACE_MS)).toEqual([]); + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]); + }); + + it("serves a drain through the window", () => { + const r: MintShardSetResolution = { set: ["a"], prevSet: ["a", "b"], flippedAtMs: T }; + expect(effectiveMintShardSet(r, T + 1, GRACE_MS)).toEqual(["a", "b"]); + expect(effectiveMintShardSet(r, T + GRACE_MS, GRACE_MS)).toEqual(["a"]); + }); +}); + +describe("readMintShardSetResolution", () => { + it("returns an empty set for an absent record", () => { + expect(readMintShardSetResolution(undefined)).toEqual({ set: [] }); + expect(readMintShardSetResolution({})).toEqual({ set: [] }); + }); + + it("reads and sorts the trio", () => { + const r = readMintShardSetResolution({ + runOpsMintShardSet: "b,a", + runOpsMintShardSetPrev: "c,a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }); + expect(r).toEqual({ set: ["a", "b"], prevSet: ["a", "c"], flippedAtMs: T }); + }); + + it("omits prevSet when no flip timestamp is stored", () => { + // A prevSet with no timestamp can never apply, so it MUST NOT linger. + const r = readMintShardSetResolution({ + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + }); + expect(r).toEqual({ set: ["a", "b"], prevSet: undefined, flippedAtMs: undefined }); + }); + + it("keeps an empty prevSet when a timestamp IS stored, which graces a first activation", () => { + const r = readMintShardSetResolution({ + runOpsMintShardSet: "a", + runOpsMintShardSetPrev: "", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }); + expect(r).toEqual({ set: ["a"], prevSet: [], flippedAtMs: T }); + }); + + it("degrades a stored value it cannot parse to an empty list instead of throwing", () => { + // Boot may throw on a bad env var. The mint path must never throw on a bad stored value. + expect(() => readMintShardSetResolution({ runOpsMintShardSet: "NOPE" })).not.toThrow(); + expect(readMintShardSetResolution({ runOpsMintShardSet: "NOPE" }).set).toEqual([]); + expect(readMintShardSetResolution({ runOpsMintShardSet: 42 }).set).toEqual([]); + expect( + readMintShardSetResolution({ + runOpsMintShardSet: "a", + runOpsMintShardSetFlippedAt: "not-a-date", + }) + ).toEqual({ set: ["a"], prevSet: undefined, flippedAtMs: undefined }); + }); +}); + +describe("stampMintShardSetFlip", () => { + it("does nothing when the save omits the set", () => { + // Omitting the set is an unrelated flag change; it must not inject a default or reset the clock. + const outgoing = { someOtherFlag: true } as Record; + expect(stampMintShardSetFlip({ runOpsMintShardSet: "a" }, outgoing, T, GRACE_MS)).toEqual({ + someOtherFlag: true, + }); + }); + + it("stamps prev and flippedAt on a genuine change", () => { + const stamped = stampMintShardSetFlip( + { runOpsMintShardSet: "a" }, + { runOpsMintShardSet: "a,b" }, + T, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("stamps an empty prev on a first activation", () => { + const stamped = stampMintShardSetFlip({}, { runOpsMintShardSet: "a" }, T, GRACE_MS); + expect(stamped.runOpsMintShardSetPrev).toBe(""); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("treats a reordered list as no change", () => { + const stamped = stampMintShardSetFlip( + { runOpsMintShardSet: "a,b" }, + { runOpsMintShardSet: "b,a" }, + T, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetFlippedAt).toBeUndefined(); + }); + + it("carries an in-flight stamp forward rather than resetting the cutover clock", () => { + const existing = { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }; + const stamped = stampMintShardSetFlip( + existing, + { runOpsMintShardSet: "a,b" }, + T + 1000, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + expect(stamped.runOpsMintShardSetFlippedAt).toBe(new Date(T).toISOString()); + }); + + it("stamps prev as the CURRENTLY-EFFECTIVE set when a second flip lands mid-window", () => { + // Two flips inside one window must not strand the original prev; prev is what readers serve now. + const existing = { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: new Date(T).toISOString(), + }; + const stamped = stampMintShardSetFlip( + existing, + { runOpsMintShardSet: "a,b,c" }, + T + 1000, + GRACE_MS + ); + expect(stamped.runOpsMintShardSetPrev).toBe("a"); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts new file mode 100644 index 00000000000..65d8af6f417 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintShardGrace.ts @@ -0,0 +1,133 @@ +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; + +// Index 24 of a gen-2 id sits inside the pod name `runner-`, and a DNS-1123 label accepts +// lowercase only, so the alphabet is 36 keys and no wider. Core keeps its copy private; +// mintShardGrace.test.ts pins this pattern to generateRunOpsIdV2 instead. +export const SHARD_KEY_PATTERN = /^[a-z0-9]$/; + +// Neither may enter the active set: "new" already means "mint a gen-1 run-ops id" and +// "legacy" means the cuid store, which minting never selects. +const RESERVED_SHARD_KEYS: readonly string[] = ["new", "legacy"]; + +// "new" IS legal as a PIN, holding one org or environment on gen-1 while the rest of the fleet +// mints gen-2. Without it a non-empty active set moves every environment at once. +export const GEN_1_PIN_VALUE = "new"; + +export type MintShardSetResolution = { + set: string[]; + prevSet?: string[]; + flippedAtMs?: number; +}; + +// Flag keys holding the active set and its grace stamp. Named here so the pure module can read +// a flag record without importing the catalog. +const SET_KEY = "runOpsMintShardSet"; +const SET_PREV_KEY = "runOpsMintShardSetPrev"; +const SET_FLIPPED_AT_KEY = "runOpsMintShardSetFlippedAt"; + +export function isValidPinValue(value: unknown): value is ShardKey { + if (typeof value !== "string") return false; + return value === GEN_1_PIN_VALUE || SHARD_KEY_PATTERN.test(value); +} + +// Throws rather than dropping a bad key: generateRunOpsIdV2 throws on an out-of-alphabet char, +// so an unvalidated key must fail at boot and never at mint. +export function parseShardCsv(raw: string | undefined | null): string[] { + const keys = (raw ?? "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + + const unique = new Set(); + for (const key of keys) { + if (RESERVED_SHARD_KEYS.includes(key)) { + throw new Error(`"${key}" is a reserved key and cannot be an active mint shard`); + } + if (!SHARD_KEY_PATTERN.test(key)) { + throw new Error(`invalid shard key "${key}": must be a single char in [a-z0-9]`); + } + unique.add(key); + } + + // Sorted so no placement can depend on the order an operator typed the CSV in. + return [...unique].sort(); +} + +// Cutover boundary, mirroring effectiveMintKind. `nowMs` is the reader's wall clock while +// `flippedAtMs` is operator-supplied, so this assumes NTP-synced hosts with skew << graceMs, +// letting every process cross [flippedAtMs, flippedAtMs + graceMs) together (OLD then NEW). +export function effectiveMintShardSet( + r: MintShardSetResolution, + nowMs: number, + graceMs: number +): string[] { + if (r.prevSet === undefined || r.flippedAtMs === undefined) { + return r.set; + } + return nowMs < r.flippedAtMs + graceMs ? r.prevSet : r.set; +} + +// The active set lives in the control-plane database, not in the environment. A deploy rolls +// for hours, so two pods can hold different environment values at the same time; only a shared +// row lets every pod agree on one set. Boot may reject a bad environment value, but the mint +// path must never throw on a bad stored value, so an unreadable list degrades to empty. +function readStoredCsv(value: unknown): string[] { + if (typeof value !== "string") return []; + try { + return parseShardCsv(value); + } catch { + return []; + } +} + +// Reads the { set, prevSet, flippedAtMs } trio out of one flag record. Pure. A prevSet with no +// timestamp can never apply, so it is dropped. A timestamp with an EMPTY prevSet is meaningful: +// it graces a first activation, serving no shards for the window. +export function readMintShardSetResolution( + flags: Record | null | undefined +): MintShardSetResolution { + const source = flags ?? {}; + const flippedAtRaw = source[SET_FLIPPED_AT_KEY]; + const parsed = typeof flippedAtRaw === "string" ? Date.parse(flippedAtRaw) : NaN; + const flippedAtMs = Number.isNaN(parsed) ? undefined : parsed; + + return { + set: readStoredCsv(source[SET_KEY]), + prevSet: flippedAtMs === undefined ? undefined : readStoredCsv(source[SET_PREV_KEY]), + flippedAtMs, + }; +} + +// Stamps a grace window only when the outgoing set differs from the stored one. prev becomes the +// set readers serve right now, so a second flip inside one window cannot strand the first. A save +// that leaves the set unchanged carries any in-flight stamp forward, so it cannot reset the clock. +export function stampMintShardSetFlip( + existingFlags: Record | null | undefined, + outgoingFlags: Record, + nowMs: number, + graceMs: number +): Record { + // Only act when the save actually SETS the list. Omitting it must not inject a default. + if (typeof outgoingFlags[SET_KEY] !== "string") { + return outgoingFlags; + } + + const existing = existingFlags ?? {}; + const outgoingSet = readStoredCsv(outgoingFlags[SET_KEY]); + const storedSet = readStoredCsv(existing[SET_KEY]); + + if (outgoingSet.join(",") !== storedSet.join(",")) { + const effective = effectiveMintShardSet(readMintShardSetResolution(existing), nowMs, graceMs); + outgoingFlags[SET_PREV_KEY] = effective.join(","); + outgoingFlags[SET_FLIPPED_AT_KEY] = new Date(nowMs).toISOString(); + return outgoingFlags; + } + + if (existing[SET_PREV_KEY] !== undefined) { + outgoingFlags[SET_PREV_KEY] = existing[SET_PREV_KEY]; + } + if (existing[SET_FLIPPED_AT_KEY] !== undefined) { + outgoingFlags[SET_FLIPPED_AT_KEY] = existing[SET_FLIPPED_AT_KEY]; + } + return outgoingFlags; +} diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts new file mode 100644 index 00000000000..542384e16f8 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -0,0 +1,92 @@ +import { $replica, boundedIn } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache"; +import { singleton } from "~/utils/singleton"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { resolveMintShardWith, type MintShardCache } from "./mintShardAssignment"; + +// A misconfiguration is reported again after this long, so a still-broken pin stays visible +// without logging on every trigger. +const REPORT_TTL_MS = 3_600_000; +const REPORT_MAX_ENTRIES = 10_000; + +const GLOBAL_SHARD_KEYS = [ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, +]; + +const liveCache = singleton("runOpsMintShardCache", (): { current: MintShardCache } => ({ + current: undefined, +})); + +async function readSetFlags(): Promise> { + const rows = await $replica.featureFlag.findMany({ + where: { key: { in: boundedIn(GLOBAL_SHARD_KEYS) } }, + select: { key: true, value: true }, + }); + const flags: Record = {}; + for (const row of rows) { + flags[row.key] = row.value; + } + return flags; +} + +// A stale pin sits on the root-trigger path, so it would otherwise log on every trigger for that +// environment forever. Bounded, because the set of pinned environments is operator-controlled but +// not operator-bounded, and an unbounded Set on this path is a leak. +const reportedPins = singleton( + "runOpsMintShardReportedPins", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); + +function reportPinRejected(info: { + environmentId: string; + pin: string; + activeSet: string[]; +}): void { + if (reportedPins.get(info.environmentId) !== undefined) return; + reportedPins.set(info.environmentId, true); + logger.error("[runOpsMintShard] pinned shard is not in the active set; using the hash", info); +} + +// Keyed by the override value, not by environment: one bad override applies to the whole fleet, +// so one line is the correct volume. Keying by environment would log once per environment. +const reportedOverrides = singleton( + "runOpsMintShardReportedOverrides", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); + +function reportOverrideRejected(info: { override: string; activeSet: string[] }): void { + if (reportedOverrides.get(info.override) !== undefined) return; + reportedOverrides.set(info.override, true); + logger.error("[runOpsMintShard] override shard is not in the active set; ignoring it", info); +} + +/** + * Which shard an environment mints new roots into. Call only after resolveRunIdMintKind has + * returned "runOpsId". Returns "new" to mean a gen-1 run-ops id, which is today's behaviour. + * + * @knipignore the gen-2 write-path change is the first production caller; drop this tag there. + */ +export async function resolveMintShard(environment: { + id: string; + // Pass environment.organization.featureFlags from the trigger call site. + orgFeatureFlags?: unknown; +}): Promise { + return resolveMintShardWith(environment, { + readFlags: readSetFlags, + cache: liveCache, + nowMs: Date.now(), + ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS, + graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, + orgFeatureFlags: environment.orgFeatureFlags, + onPinRejected: reportPinRejected, + onOverrideRejected: reportOverrideRejected, + onReadFailed: (error) => + logger.error("[runOpsMintShard] shard-set read failed; minting gen-1 (fail-safe)", { error }), + }); +} diff --git a/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts index a510fbe0f35..b5bfd8ea052 100644 --- a/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts +++ b/apps/webapp/test/adminFeatureFlagsRouteAction.test.ts @@ -2,7 +2,7 @@ // bug surface. These drive the real exported action against a real Postgres and assert on the rows // it leaves behind. The only module substituted is the auth wrapper, so the handler can be called // without a super-admin session; the database is the genuine article, injected into db.server. -import { boundedIn } from "@trigger.dev/database"; +import { boundedIn, $transaction as realTransaction } from "@trigger.dev/database"; import type { PrismaClient } from "@trigger.dev/database"; import { postgresTest } from "@internal/testcontainers"; import { describe, expect, vi } from "vitest"; @@ -22,6 +22,24 @@ vi.mock("~/db.server", () => ({ return db.client; }, boundedIn, + // Delegates to the SAME shared implementation the production helper wraps, so the + // transactional semantics, the nesting case and the retry behaviour are the real ones rather + // than a reimplementation. Only the webapp wrapper's tracing span and its infrastructure-error + // logging are absent, and neither is asserted here. + $transaction: ( + client: PrismaClient, + nameOrFn: unknown, + fnOrOptions?: unknown, + options?: unknown + ) => { + const fn = (typeof nameOrFn === "function" ? nameOrFn : fnOrOptions) as Parameters< + typeof realTransaction + >[1]; + const opts = (typeof nameOrFn === "function" ? fnOrOptions : options) as Parameters< + typeof realTransaction + >[3]; + return realTransaction(client, fn, () => {}, opts); + }, })); import { action } from "~/routes/admin.feature-flags"; diff --git a/apps/webapp/test/globalFlagChangeList.test.ts b/apps/webapp/test/globalFlagChangeList.test.ts new file mode 100644 index 00000000000..269774e28d3 --- /dev/null +++ b/apps/webapp/test/globalFlagChangeList.test.ts @@ -0,0 +1,142 @@ +// Two properties of a global flag save that the admin page had no way to state. +// +// 1. Unsetting a graced primary clears its server-computed stamps too. Those keys are locked, so +// they are absent from the page's editable set, and the confirm dialog listed one removal +// while three rows were deleted. +// 2. A save should write only the flags whose value actually changed. Writing every submitted +// flag costs one round trip each inside an interactive transaction. +import { describe, expect, it } from "vitest"; +import { FEATURE_FLAG, derivedFlagsClearedWith } from "~/v3/featureFlags"; +import { flagsNeedingWrite } from "~/v3/featureFlags.server"; +import { buildFlagChangeList } from "~/components/admin/flagChangeList"; + +const LOCKED = [ + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, +] as string[]; + +// Sorted, as the dialog sorts before calling: the builder preserves the order it is given. +const EDITABLE = [ + FEATURE_FLAG.runOpsMintKind, + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.mollifierEnabled, +].sort() as string[]; + +describe("derivedFlagsClearedWith", () => { + it("names the stamps that go with a graced primary", () => { + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintKind)).toEqual([ + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + ]); + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintShardSet)).toEqual([ + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + }); + + it("names nothing for an ordinary flag, or for a stamp itself", () => { + expect(derivedFlagsClearedWith(FEATURE_FLAG.mollifierEnabled)).toEqual([]); + expect(derivedFlagsClearedWith(FEATURE_FLAG.runOpsMintKindPrev)).toEqual([]); + }); +}); + +describe("buildFlagChangeList — what the confirm dialog must show", () => { + it("lists an added, a changed and a removed flag", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { mollifierEnabled: true, runOpsMintShardSet: "a" }, + storedValues: { mollifierEnabled: true, runOpsMintShardSet: "a" }, + newValues: { runOpsMintShardSet: "a,b", runOpsMintKind: "runOpsId" }, + }); + + expect(changes).toEqual([ + { key: FEATURE_FLAG.mollifierEnabled, type: "removed", oldVal: "true" }, + { key: FEATURE_FLAG.runOpsMintKind, type: "added", newVal: "runOpsId" }, + { key: FEATURE_FLAG.runOpsMintShardSet, type: "changed", oldVal: "a", newVal: "a,b" }, + ]); + }); + + it("discloses the stamps cleared alongside an unset graced primary", () => { + // Three rows are deleted, so three removals must be shown, not one. The caller filters + // locked keys OUT of initialValues, so the stamps are only visible in storedValues. + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a,b" }, + storedValues: { + runOpsMintShardSet: "a,b", + runOpsMintShardSetPrev: "a", + runOpsMintShardSetFlippedAt: "2026-08-24T00:00:00.000Z", + }, + newValues: {}, + }); + + expect(changes.map((c) => c.key)).toEqual([ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + expect(changes.every((c) => c.type === "removed")).toBe(true); + }); + + it("does not disclose a stamp that is not stored", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a,b" }, + storedValues: { runOpsMintShardSet: "a,b" }, + newValues: {}, + }); + expect(changes.map((c) => c.key)).toEqual([FEATURE_FLAG.runOpsMintShardSet]); + }); + + it("does not disclose stamps when the primary is only CHANGED", () => { + // A change re-stamps rather than clearing, so nothing is removed. + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: { runOpsMintShardSet: "a" }, + storedValues: { runOpsMintShardSet: "a", runOpsMintShardSetPrev: "" }, + newValues: { runOpsMintShardSet: "a,b" }, + }); + expect(changes.map((c) => c.key)).toEqual([FEATURE_FLAG.runOpsMintShardSet]); + }); + + it("never lists a locked key on its own", () => { + const changes = buildFlagChangeList({ + editableKeys: EDITABLE, + lockedKeys: LOCKED, + initialValues: {}, + storedValues: { runOpsMintShardSetPrev: "a" }, + newValues: {}, + }); + expect(changes).toEqual([]); + }); +}); + +describe("flagsNeedingWrite — one round trip per CHANGED flag, not per submitted flag", () => { + it("drops a submitted flag whose stored value already matches", () => { + const out = flagsNeedingWrite( + { mollifierEnabled: true, hasAiAccess: true }, + { mollifierEnabled: true, hasAiAccess: false } + ); + expect(out).toEqual({ hasAiAccess: true }); + }); + + it("keeps a flag that is absent from storage", () => { + expect(flagsNeedingWrite({ mollifierEnabled: true }, {})).toEqual({ mollifierEnabled: true }); + }); + + it("returns nothing when a save changes nothing", () => { + expect(flagsNeedingWrite({ mollifierEnabled: true }, { mollifierEnabled: true })).toEqual({}); + }); + + it("compares by value, not by reference, so a CSV rewritten the same way is not a write", () => { + expect(flagsNeedingWrite({ runOpsMintShardSet: "a,b" }, { runOpsMintShardSet: "a,b" })).toEqual( + {} + ); + }); +}); diff --git a/apps/webapp/test/globalFlagWriteRouting.test.ts b/apps/webapp/test/globalFlagWriteRouting.test.ts new file mode 100644 index 00000000000..3193919b5ca --- /dev/null +++ b/apps/webapp/test/globalFlagWriteRouting.test.ts @@ -0,0 +1,110 @@ +// Both global write routes used to carry their own copy of "which keys are graced" and "which +// keys are derived", so a new group needed an edit in three places and missing one meant an +// unstamped flip or a stamp taken from a request body. These tests cover the two helpers the +// routes now call. They do NOT reach a route: both actions sit behind admin auth, so that the +// routes call these helpers rather than their own copies is held by review, not by a test. +import { describe, expect, it } from "vitest"; +import { FEATURE_FLAG, lockedFlagsInPayload } from "~/v3/featureFlags"; +import { touchesGracedGroup, withoutDerivedKeys } from "~/v3/featureFlags.server"; + +describe("touchesGracedGroup — decides whether a save needs the stamped path", () => { + it("is true for a mint-kind change", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintKind]: "runOpsId" })).toBe(true); + }); + + it("is true for a shard-list change", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintShardSet]: "a,b" })).toBe(true); + }); + + it("is false for an ordinary flag, which writes directly", () => { + expect(touchesGracedGroup({ [FEATURE_FLAG.mollifierEnabled]: true })).toBe(false); + expect(touchesGracedGroup({})).toBe(false); + }); + + it("is false when only a DERIVED key is present", () => { + // A body carrying only a stamp changes no group. Treating it as a flip would let a caller + // reset a cutover clock without touching the value the clock dates. + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintKindPrev]: "cuid" })).toBe(false); + expect(touchesGracedGroup({ [FEATURE_FLAG.runOpsMintShardSetPrev]: "a" })).toBe(false); + }); + + it("recognises every graced primary the group table declares", () => { + const gracedPrimaries = [FEATURE_FLAG.runOpsMintKind, FEATURE_FLAG.runOpsMintShardSet]; + for (const key of gracedPrimaries) { + expect(touchesGracedGroup({ [key]: "x" })).toBe(true); + } + // Every key the strip removes belongs to a group whose primary is one of the above. + const derived = Object.keys( + withoutDerivedKeys({ + [FEATURE_FLAG.runOpsMintKindPrev]: "cuid", + [FEATURE_FLAG.runOpsMintKindFlippedAt]: "t", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "a", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "t", + } as Record) + ); + expect(derived).toEqual([]); + }); +}); + +describe("withoutDerivedKeys — a stamp is never taken from a request body", () => { + it("strips both stamps and keeps everything else", () => { + const out = withoutDerivedKeys({ + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintKindPrev]: "spoofed", + [FEATURE_FLAG.runOpsMintKindFlippedAt]: "1999-01-01T00:00:00.000Z", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "spoofed", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "1999-01-01T00:00:00.000Z", + [FEATURE_FLAG.mollifierEnabled]: true, + } as Record); + + expect(out).toEqual({ + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.mollifierEnabled]: true, + }); + }); + + it("does not mutate its input", () => { + const input = { [FEATURE_FLAG.runOpsMintKindPrev]: "cuid" } as Record; + withoutDerivedKeys(input); + expect(input[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid"); + }); +}); + +describe("lockedFlagsInPayload — what the global page refuses", () => { + it("refuses a locked flag on managed cloud, where the page never offers one", () => { + const refused = lockedFlagsInPayload( + [FEATURE_FLAG.taskEventRepository, FEATURE_FLAG.mollifierEnabled], + true + ); + expect(refused).toEqual([FEATURE_FLAG.taskEventRepository]); + }); + + it("refuses the mint-shard pins, which are per-org only", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShard], true)).toEqual([ + FEATURE_FLAG.runOpsMintShard, + ]); + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardEnvPins], true)).toEqual([ + FEATURE_FLAG.runOpsMintShardEnvPins, + ]); + }); + + it("refuses a grace stamp, which the server owns", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardSetFlippedAt], true)).toEqual([ + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + ]); + }); + + it("allows the shard list, because that is the page's ramp lever", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.runOpsMintShardSet], true)).toEqual([]); + }); + + it("refuses nothing when not managed cloud, where an admin may unlock and edit", () => { + expect(lockedFlagsInPayload([FEATURE_FLAG.taskEventRepository], false)).toEqual([]); + }); + + it("refuses nothing for an empty payload", () => { + expect(lockedFlagsInPayload([], true)).toEqual([]); + }); +}); diff --git a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts index 1492fd02c43..b3acd7e45eb 100644 --- a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts +++ b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts @@ -5,7 +5,7 @@ import type { PrismaClient } from "@trigger.dev/database"; import { postgresTest } from "@internal/testcontainers"; import { describe, expect, vi } from "vitest"; import { FEATURE_FLAG } from "~/v3/featureFlags"; -import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { applyGlobalGracedFlips, makeSetMultipleFlags } from "~/v3/featureFlags.server"; vi.setConfig({ testTimeout: 60_000 }); @@ -25,11 +25,11 @@ async function readGlobalMint(prisma: PrismaClient): Promise { +describe("applyGlobalGracedFlips — transactional stamp + serialized flips", () => { postgresTest("a genuine global flip stamps prev + flippedAt", async ({ prisma }) => { await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" }); - await applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); const m = await readGlobalMint(prisma); expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId"); @@ -44,7 +44,7 @@ describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", ( await Promise.all( Array.from({ length: 8 }, () => - applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) ) ); diff --git a/apps/webapp/test/runOpsMintShardFlags.test.ts b/apps/webapp/test/runOpsMintShardFlags.test.ts new file mode 100644 index 00000000000..98b76eb19c8 --- /dev/null +++ b/apps/webapp/test/runOpsMintShardFlags.test.ts @@ -0,0 +1,118 @@ +// The mint-shard flags carry two safety claims that only the catalog can enforce: a bad value is +// rejected at WRITE (so no unroutable key and no silently-unpinned environment can ever be +// stored), and each key is locked at the scope its resolver does not read. Pure, no containers. +import { describe, expect, it } from "vitest"; +import { + FEATURE_FLAG, + FeatureFlagCatalog, + GLOBAL_LOCKED_FLAGS, + ORG_LOCKED_FLAGS, + validateFeatureFlagValue, +} from "~/v3/featureFlags"; + +describe("runOpsMintShard — the per-org pin", () => { + const key = FEATURE_FLAG.runOpsMintShard; + + it("accepts every legal shard key", () => { + for (const c of "abcdefghijklmnopqrstuvwxyz0123456789") { + expect(validateFeatureFlagValue(key, c).success).toBe(true); + } + }); + + it('accepts "new", which holds an org on gen-1', () => { + expect(validateFeatureFlagValue(key, "new").success).toBe(true); + }); + + it("rejects a value that could never be stamped into an id", () => { + for (const bad of ["A", "ab", "", "-", "legacy", " a", "a,b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardEnvPins — the per-environment pins", () => { + const key = FEATURE_FLAG.runOpsMintShardEnvPins; + + it("accepts a map of environment id to shard key", () => { + expect( + validateFeatureFlagValue(key, JSON.stringify({ env_1: "a", env_2: "new" })).success + ).toBe(true); + expect(validateFeatureFlagValue(key, "{}").success).toBe(true); + }); + + it("rejects a blob that is not JSON, so a typo cannot silently un-pin every environment", () => { + for (const bad of ["{not json", "", "null", "[]", '"a"', "42"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); + + it("rejects a map whose value is not a legal pin", () => { + for (const bad of [{ env_1: "AB" }, { env_1: "legacy" }, { env_1: 1 }, { env_1: "" }]) { + expect(validateFeatureFlagValue(key, JSON.stringify(bad)).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardSet — the active list", () => { + const key = FEATURE_FLAG.runOpsMintShardSet; + + it("accepts an empty list and a CSV of legal keys", () => { + expect(validateFeatureFlagValue(key, "").success).toBe(true); + expect(validateFeatureFlagValue(key, "a").success).toBe(true); + expect(validateFeatureFlagValue(key, "a,b, c").success).toBe(true); + }); + + it("rejects a CSV holding a key that cannot be routed", () => { + for (const bad of ["A", "ab", "a,B", "a,legacy", "a,new", "a;b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("runOpsMintShardOverride — the complete-cutover lever", () => { + const key = FEATURE_FLAG.runOpsMintShardOverride; + + it("accepts a shard key and accepts new", () => { + expect(validateFeatureFlagValue(key, "a").success).toBe(true); + expect(validateFeatureFlagValue(key, "new").success).toBe(true); + }); + + it("rejects anything that is not a single legal key", () => { + for (const bad of ["A", "ab", "", "legacy", "a,b"]) { + expect(validateFeatureFlagValue(key, bad).success).toBe(false); + } + }); +}); + +describe("scope locks match what each resolver actually reads", () => { + it("locks the pins globally, because the resolver reads them from the org blob only", () => { + expect(GLOBAL_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShard); + expect(GLOBAL_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardEnvPins); + }); + + it("locks the list and the override per-org, because both are deployment-wide", () => { + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSet); + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSetPrev); + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardSetFlippedAt); + // An org that could override the cutover lever would defeat its purpose. + expect(ORG_LOCKED_FLAGS).toContain(FEATURE_FLAG.runOpsMintShardOverride); + }); + + it("keeps the pins settable per-org, which is the canary lever", () => { + expect(ORG_LOCKED_FLAGS).not.toContain(FEATURE_FLAG.runOpsMintShard); + expect(ORG_LOCKED_FLAGS).not.toContain(FEATURE_FLAG.runOpsMintShardEnvPins); + }); + + it("registers every new key in the catalog, so the admin pages render it", () => { + for (const key of [ + FEATURE_FLAG.runOpsMintShard, + FEATURE_FLAG.runOpsMintShardEnvPins, + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, + FEATURE_FLAG.runOpsMintShardOverride, + ]) { + expect(FeatureFlagCatalog).toHaveProperty(key); + } + }); +}); diff --git a/apps/webapp/test/runOpsMintShardSetFlip.test.ts b/apps/webapp/test/runOpsMintShardSetFlip.test.ts new file mode 100644 index 00000000000..6dc126fc0bd --- /dev/null +++ b/apps/webapp/test/runOpsMintShardSetFlip.test.ts @@ -0,0 +1,279 @@ +// The active mint-shard list lives in the control-plane database, not in the environment: a +// rolling deploy runs two environment values at once for hours, so only a shared row lets every +// pod agree on one list. A change must therefore read -> stamp -> write under an advisory lock, +// and must never be writable as a bare upsert from a request body. Real testcontainers Postgres. +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG, type FeatureFlagKey } from "~/v3/featureFlags"; +import { + applyGlobalGracedFlips, + makeSetMultipleFlags, + replaceGlobalFeatureFlags, +} from "~/v3/featureFlags.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const SET_KEYS: FeatureFlagKey[] = [ + FEATURE_FLAG.runOpsMintShardSet, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, +]; + +const MINT_KIND_KEYS: FeatureFlagKey[] = [ + FEATURE_FLAG.runOpsMintKind, + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, +]; + +const CATALOG_KEYS: FeatureFlagKey[] = [ + ...SET_KEYS, + ...MINT_KIND_KEYS, + FEATURE_FLAG.mollifierEnabled, +]; + +// Self-hosted with the lock left on: locked flags survive omission, everything else sweeps. +const SELF_HOSTED = { isManagedCloud: false, unlockLockedFlags: false } as const; + +async function readFlags( + prisma: PrismaClient, + keys: FeatureFlagKey[] +): Promise> { + const rows = await prisma.featureFlag.findMany({ + where: { key: { in: keys } }, + select: { key: true, value: true }, + }); + const m: Record = {}; + for (const row of rows) m[row.key] = row.value; + return m; +} + +describe("applyGlobalGracedFlips — the shard-set list is stamped, not bare-written", () => { + postgresTest("a genuine list change stamps prev + flippedAt", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest("a first activation stamps an empty prev, which graces it", async ({ prisma }) => { + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a" }, 60_000); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe(""); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest( + "a reordered list is not a change, so the clock is not reset", + async ({ prisma }) => { + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000); + const first = await readFlags(prisma, SET_KEYS); + + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "b,a" }, 60_000); + const second = await readFlags(prisma, SET_KEYS); + + expect(second[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe( + first[FEATURE_FLAG.runOpsMintShardSetFlippedAt] + ); + } + ); + + postgresTest("both graced groups stamp in ONE save", async ({ prisma }) => { + // A save that flips the kind and the list must not stamp one and lose the other. + await makeSetMultipleFlags(prisma)({ + [FEATURE_FLAG.runOpsMintKind]: "cuid", + [FEATURE_FLAG.runOpsMintShardSet]: "a", + }); + + await applyGlobalGracedFlips( + prisma, + { + [FEATURE_FLAG.runOpsMintKind]: "runOpsId", + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + }, + 60_000 + ); + + const m = await readFlags(prisma, [...SET_KEYS, ...MINT_KIND_KEYS]); + expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(typeof m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBe("string"); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); + + postgresTest("concurrent list changes do not interleave", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await Promise.all([ + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, 60_000), + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintShardSet]: "a,c" }, 60_000), + ]); + + const m = await readFlags(prisma, SET_KEYS); + const set = m[FEATURE_FLAG.runOpsMintShardSet]; + const prev = m[FEATURE_FLAG.runOpsMintShardSetPrev]; + + // The pair must be a consistent history, not a mix of the two writers. The winner's set is + // one of the two, and prev is what the OTHER writer left behind: either the original "a", or + // the loser's set when the loser committed first. "a,b" beside prev "a,b" would mean one + // writer read its own uncommitted state, and prev naming the winner's own set is incoherent. + expect(["a,b", "a,c"]).toContain(set); + expect(["a", "a,b", "a,c"]).toContain(prev); + expect(prev).not.toBe(set); + expect(typeof m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe("string"); + }); +}); + +describe("replaceGlobalFeatureFlags — the admin page cannot bypass the stamp", () => { + postgresTest("a list change through the page is stamped", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + }); + + postgresTest("a body-supplied stamp is ignored and recomputed", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintShardSet]: "a" }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.runOpsMintShardSetPrev]: "zzz", + [FEATURE_FLAG.runOpsMintShardSetFlippedAt]: "1999-01-01T00:00:00.000Z", + }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBe("a"); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).not.toBe("1999-01-01T00:00:00.000Z"); + }); + + postgresTest("a co-submitted flag does not disturb a resubmitted list", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + const first = await readFlags(prisma, SET_KEYS); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { + [FEATURE_FLAG.runOpsMintShardSet]: "a,b", + [FEATURE_FLAG.mollifierEnabled]: true, + }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + // Resubmitting the same list is not a flip, so the cutover clock is not reset. + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBe( + first[FEATURE_FLAG.runOpsMintShardSetFlippedAt] + ); + }); + + postgresTest( + "omitting the list DELETES it, so unset still turns minting off", + async ({ prisma }) => { + // The admin page's unset button omits the key. If the save skipped it, unset would be a + // silent no-op and gen-2 minting would stay armed. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + // The stamp goes with it: a stamp without its list keeps being served for the whole window. + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeUndefined(); + } + ); + + postgresTest("omitting the mint kind still deletes its trio", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, MINT_KIND_KEYS); + expect(m[FEATURE_FLAG.runOpsMintKind]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBeUndefined(); + }); + + postgresTest("unlocking does not orphan a stamp from its list", async ({ prisma }) => { + // With the lock off, a locked key may be swept. The stamps are locked, so this is the case + // where they could be deleted while the list survives, which would keep serving prevSet. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + isManagedCloud: false, + unlockLockedFlags: true, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeDefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeDefined(); + }); + + postgresTest("an ordinary flag keeps replace semantics", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + ...SELF_HOSTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, [FEATURE_FLAG.mollifierEnabled]); + expect(m[FEATURE_FLAG.mollifierEnabled]).toBeUndefined(); + }); +}); diff --git a/knip.json b/knip.json index 84456756ca1..c6e8aee8977 100644 --- a/knip.json +++ b/knip.json @@ -25,7 +25,8 @@ "vite/node-globals-shim.js", "app/v3/otlpTransformWorker.ts" ], - "ignoreDependencies": ["@sentry/cli", "assert", "util"] + "ignoreDependencies": ["@sentry/cli", "assert", "util"], + "ignore": ["app/v3/runOpsMigration/runOpsMintShard.server.ts"] }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], From cc69ff4d267fe3e6bb3e4f41f49b9bb38afe1001 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:40:25 +0100 Subject: [PATCH 06/28] feat(run-engine): Redis waitpoint store coordinator, Lua protocol, and waitpoint ids (#4761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the Redis-backed half of the waitpoint coordinator, beside the Postgres coordinator that #4753 extracted. Adds the coordination protocol as Lua scripts, the run-ops-format waitpoint id scheme, and the key layout. **No caller wires any of it up.** Refs TRI-13440. ## Inert by construction Merging this changes nothing observable. 3180 insertions, **zero deletions**, nine new or additively-edited files. - `WaitpointStoreCoordinator` is never constructed outside its own tests and the benchmark. - No env var, no config plumbing, no connection. It takes `redisOptions` as a constructor argument. - `waitpointSystem.ts` is untouched. Every live waitpoint operation still runs on Postgres through the coordinator merged in #4753. - No changeset and no `.server-changes` note — nothing here is user-facing yet. Deploying this needs no Redis or MemoryDB instance. That becomes a prerequisite when a later change routes traffic onto the store behind a per-organisation flag. ## What's here **Nine Lua scripts**, each atomic on one hash tag. Seven mutate state — create-if-absent, register-or-report, complete, idempotency reserve, absorb, deliver, clear. One reads state (`runReadBlockState`) and is separate because the pending, delivered and edge sets must be read as one consistent view. One discards an idempotency loser. **Two hash tags, deliberately.** `wp:{waitpointId}` holds a waitpoint's record, status, completion envelope and watcher hash. `wp:run:{runId}:*` holds one run's pending set, delivered set and edge set. A waitpoint has N watchers, so it cannot live under any single run's tag. **Waitpoint ids** reuse the run-ops body layout: a 24-char base32hex core, a type char (`r`/`b`/`d`/`m`), and version char `w`. RUN and BATCH ids derive from their anchor's core, so create-if-absent is idempotent with no lock. `parseWaitpointId` is total and never throws. **The single-slot guard.** Every script invocation goes through one private wrapper that asserts all keys share a hash tag. A single-node test server accepts what a real cluster rejects, so this assertion is the only enforcement — and it is mutation-tested: removing it fails a test. ## Measured Against the same population of real Postgres rows: | | store | postgres | |---|---|---| | pending count (the blocked/unblocked gate) | 0.13 ms p50 | 3.32 ms p50 | | full-payload read | 1.45 ms p50 | 7.70 ms p50 | Both are lower bounds: the benchmark charges Postgres a `COUNT(*)`, while the resume-time read is a join with a partial select plus filtering in JavaScript. Store-only paths, no Postgres counterpart: block+complete+deliver 0.88 ms p50; 100-watcher fan-out 13.8 ms; a 1001-edge fan-in 149.8 ms, flat at 0.15 ms per edge and round-trip bound rather than algorithmic. The benchmark lives in `*.bench.test.ts` and is excluded from the default suite. ## Review notes - **The type surfaces are not reconciled yet, on purpose.** `types.ts` (from #4753) carries the coordinator interface; `storeCoordinator.ts` declares its own operation types because this was built in parallel. The wiring change reconciles them. - **The read-time resolver is not here.** Another lane froze its contract while this was in flight, and its frozen types are not yet on main. Building a second copy would fork a just-frozen contract. - **Teardown is one-shard while registration is two-shard.** A terminal clear leaves a run registered as a watcher on the waitpoints it was blocked on, because the watcher hash is under a different tag and no script may span slots. Recorded, not fixed here — it needs a retention decision, and nothing observes it while the code is unwired. ## Verification 79 tests in the coordinator suite, 58 in the id suite. `typecheck` on run-engine and webapp, `build` on core, `knip`, `oxfmt` and `oxlint` all clean. The engine corpus passes 82/82. Every invariant is mutation-tested rather than merely asserted. A whole-branch review ran 14 mutants and killed 12; the two survivors were fixed with their own mutation checks. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../bench/waitpointCoordinator.bench.test.ts | 266 +++ .../engine/waitpointCoordinator/keys.test.ts | 132 ++ .../src/engine/waitpointCoordinator/keys.ts | 98 + .../engine/waitpointCoordinator/scripts.ts | 388 ++++ .../storeCoordinator.test.ts | 1837 +++++++++++++++++ .../waitpointCoordinator/storeCoordinator.ts | 538 +++++ internal-packages/run-engine/src/index.ts | 23 + .../core/src/v3/isomorphic/friendlyId.test.ts | 162 ++ packages/core/src/v3/isomorphic/friendlyId.ts | 99 + 9 files changed, 3543 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts diff --git a/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts new file mode 100644 index 00000000000..01393ba4f96 --- /dev/null +++ b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts @@ -0,0 +1,266 @@ +/** + * Waitpoint coordination benchmark. Reports numbers; asserts nothing — on a shared runner + * the timings swing far more than any threshold worth gating on. + * + * Four groups, and only the first two are pairs: + * + * 1. Pending count — the store's SCARD gate against the previous path's + * `COUNT(*) ... WHERE status='PENDING'`, over the same population. Like for like. + * 2. Read amplification — the store's `readBlockState` against a full-payload `SELECT` + * of the same waitpoints. Like for like. + * 3. Store-only write paths — block+complete+deliver and K-watcher fan-out. Absolute + * numbers with NO Postgres counterpart: no single statement on the previous path + * corresponds to a Redis round trip that both blocks a run and delivers to watchers. + * 4. Register cost versus edge count — `registerBlocks` registers each edge with its own + * round trip before the single absorb. This measures whether that serial loop is a + * real cost at a wide fan-in, or a non-issue, at several fan-in widths. + * + * Every Postgres measurement here runs against rows this file inserts. A baseline over an + * empty table measures nothing. + * + * Knobs: BENCH_WP_ITERATIONS, BENCH_WP_FANIN, BENCH_WP_WATCHERS, BENCH_WP_REGISTER_WIDTHS, + * BENCH_WP_REGISTER_SAMPLES. + */ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { + WaitpointStoreCoordinator, + type BlockEdge, + type WaitpointRecordInput, +} from "../waitpointCoordinator/storeCoordinator.js"; +import { setupAuthenticatedEnvironment } from "../tests/setup.js"; + +vi.setConfig({ testTimeout: 900_000 }); + +const ITERATIONS = Number(process.env.BENCH_WP_ITERATIONS ?? 100); +const FANIN = Number(process.env.BENCH_WP_FANIN ?? 1001); +const WATCHERS = Number(process.env.BENCH_WP_WATCHERS ?? 100); +const REGISTER_WIDTHS = (process.env.BENCH_WP_REGISTER_WIDTHS ?? "1,10,100,1001") + .split(",") + .map((raw) => Number(raw.trim())) + .filter((width) => Number.isFinite(width) && width > 0); +const REGISTER_SAMPLES = Number(process.env.BENCH_WP_REGISTER_SAMPLES ?? 20); +const NOW = new Date().toISOString(); + +type Sample = { label: string; count: number; p50: number; p99: number; totalMs: number }; + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]!; +} + +async function measure(label: string, count: number, run: (i: number) => Promise) { + const durations: number[] = []; + const started = Date.now(); + for (let i = 0; i < count; i++) { + const t0 = performance.now(); + await run(i); + durations.push(performance.now() - t0); + } + durations.sort((a, b) => a - b); + const sample: Sample = { + label, + count, + p50: percentile(durations, 50), + p99: percentile(durations, 99), + totalMs: Date.now() - started, + }; + console.log( + `[bench] ${sample.label} n=${sample.count} p50=${sample.p50.toFixed(2)}ms ` + + `p99=${sample.p99.toFixed(2)}ms total=${sample.totalMs}ms` + ); + return sample; +} + +function record(id: string, environmentId: string, projectId: string): WaitpointRecordInput { + return { + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL", + environmentId, + projectId, + createdAt: NOW, + updatedAt: NOW, + userProvidedIdempotencyKey: false, + tags: [], + }; +} + +const completion = { + completedAt: NOW, + outputType: "application/json", + outputIsError: false, + output: { inline: '{"ok":true}' }, +}; + +function edge(waitpointId: string, batchIndex?: number): BlockEdge { + return { waitpointId, batchIndex, createdAt: NOW, type: "MANUAL" }; +} + +async function insertWaitpoints( + prisma: PrismaClient, + ids: string[], + environmentId: string, + projectId: string +) { + await prisma.waitpoint.createMany({ + data: ids.map((id) => ({ + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL" as const, + idempotencyKey: id, + userProvidedIdempotencyKey: false, + projectId, + environmentId, + })), + }); +} + +containerTest( + "waitpoint coordination: pending count, read amplification, store write paths, register cost", + async ({ prisma, redisOptions }) => { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const store = new WaitpointStoreCoordinator({ redisOptions }); + const samples: Sample[] = []; + const registerCost: Array<{ + width: number; + p50Ms: number; + p99Ms: number; + perEdgeMsP50: number; + }> = []; + + try { + const ids = Array.from({ length: FANIN }, (_, i) => `bench_w_${i}`); + + // Both stores get the SAME population. A Postgres baseline over an empty table + // measures an index probe against nothing. + await insertWaitpoints(prisma, ids, env.id, env.project.id); + for (const id of ids) { + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + } + await store.registerBlocks({ + runId: "bench_run_fanin", + edges: ids.map((id, index) => edge(id, index)), + }); + + // --- group 1: the pending-count gate, like for like --- + samples.push( + await measure("store.pendingCount", ITERATIONS, async () => { + await store.absorbBlockers({ runId: "bench_run_fanin", edges: [] }); + }) + ); + samples.push( + await measure("postgres.pendingCount", ITERATIONS, async () => { + await prisma.$queryRaw`SELECT COUNT(*) FROM "Waitpoint" WHERE id = ANY(${ids}::text[]) AND status = 'PENDING'`; + }) + ); + + // --- group 2: read amplification, like for like --- + samples.push( + await measure("store.readBlockState", ITERATIONS, async () => { + await store.readBlockState("bench_run_fanin"); + }) + ); + samples.push( + await measure("postgres.hydrateFullPayload", ITERATIONS, async () => { + // Every column of every waitpoint — the amplification the store removes. + await prisma.waitpoint.findMany({ where: { id: { in: ids } } }); + }) + ); + + // --- group 3: store-only write paths, no Postgres counterpart --- + samples.push( + await measure("store.block+complete+deliver", ITERATIONS, async (i) => { + const id = `bench_cycle_${i}`; + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + await store.registerBlocks({ runId: `bench_run_${i}`, edges: [edge(id)] }); + const done = await store.complete({ waitpointId: id, completion }); + for (const watcher of done.watchers) { + await store.deliverCompletion({ + runId: watcher.runId, + waitpointId: id, + completion: done.completion!, + }); + } + }) + ); + + const fanOutId = "bench_fanout_w"; + await store.createIfAbsent({ + record: record(fanOutId, env.id, env.project.id), + status: "PENDING", + }); + for (let i = 0; i < WATCHERS; i++) { + await store.registerBlocks({ runId: `bench_watcher_${i}`, edges: [edge(fanOutId)] }); + } + samples.push( + await measure(`store.complete+deliver(watchers=${WATCHERS})`, 1, async () => { + const done = await store.complete({ waitpointId: fanOutId, completion }); + // Serial on purpose: this is the worst case, and it is the number that says + // whether delivery needs to pipeline. + for (const watcher of done.watchers) { + await store.deliverCompletion({ + runId: watcher.runId, + waitpointId: fanOutId, + completion: done.completion!, + }); + } + }) + ); + + // --- group 4: register cost versus edge count --- + // registerBlocks registers each edge with its own round trip, serially, before the + // single absorb. A review flagged that a wide fan-in therefore serializes one round + // trip per edge. This measures the real cost at several widths rather than predicting + // it, so the decision about bounded concurrency is made against a number. + const registerPoolWidth = Math.max(0, ...REGISTER_WIDTHS); + const registerIds = Array.from({ length: registerPoolWidth }, (_, i) => `bench_reg_w_${i}`); + await insertWaitpoints(prisma, registerIds, env.id, env.project.id); + for (const id of registerIds) { + await store.createIfAbsent({ + record: record(id, env.id, env.project.id), + status: "PENDING", + }); + } + + for (const width of REGISTER_WIDTHS) { + const edges = registerIds.slice(0, width).map((id, index) => edge(id, index)); + let call = 0; + const sample = await measure( + `store.registerBlocks(edges=${width})`, + REGISTER_SAMPLES, + async () => { + await store.registerBlocks({ runId: `bench_register_${width}_${call++}`, edges }); + } + ); + samples.push(sample); + registerCost.push({ + width, + p50Ms: sample.p50, + p99Ms: sample.p99, + perEdgeMsP50: sample.p50 / width, + }); + console.log( + `[bench] store.registerBlocks(edges=${width}) implied per-edge cost ` + + `p50=${(sample.p50 / width).toFixed(3)}ms p99=${(sample.p99 / width).toFixed(3)}ms` + ); + } + + console.log( + `[bench] groups 1 and 2 are like-for-like pairs. Group 3 and the register-cost ` + + `group (4) have no Postgres counterpart: no single statement on the previous ` + + `path corresponds to a Redis round trip that blocks, completes and delivers, ` + + `or to a serial per-edge register loop.` + ); + console.log(`[bench] summary\n${JSON.stringify({ samples, registerCost }, null, 2)}`); + } finally { + await store.quit(); + } + } +); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts new file mode 100644 index 00000000000..463e1ecd388 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { + WaitpointKeyTagError, + assertSingleSlot, + edgeField, + idempotencyKey, + runBlockKeys, + waitpointIdFromEdgeField, + waitpointKeys, + watcherField, +} from "./keys.js"; + +describe("waitpointKeys", () => { + it("puts the record and its watchers under one hash tag", () => { + const k = waitpointKeys("abc123w"); + expect(k.record).toBe("wp:{abc123w}"); + expect(k.watchers).toBe("wp:{abc123w}:w"); + }); +}); + +describe("runBlockKeys", () => { + it("puts all three run keys under one hash tag", () => { + const k = runBlockKeys("run_abc"); + expect(k.pend).toBe("wp:run:{run_abc}:pend"); + expect(k.done).toBe("wp:run:{run_abc}:done"); + expect(k.edge).toBe("wp:run:{run_abc}:edge"); + }); +}); + +describe("idempotencyKey", () => { + it("tags by environment, so one environment's reservations share a slot", () => { + expect(idempotencyKey("env_1", "my-key")).toBe("wp:idem:{env_1}:my-key"); + }); +}); + +describe("edgeField", () => { + it("keys by waitpoint id and batch index, matching the Postgres unique key", () => { + expect(edgeField("w_a", 3)).toBe("w_a#3"); + }); + + it("collapses a null or absent batch index onto one field", () => { + expect(edgeField("w_a")).toBe("w_a#"); + expect(edgeField("w_a", null)).toBe("w_a#"); + }); + + it("distinguishes index 0 from an absent index", () => { + expect(edgeField("w_a", 0)).not.toBe(edgeField("w_a")); + }); +}); + +describe("waitpointIdFromEdgeField", () => { + it("round-trips back to the waitpoint id", () => { + for (const index of [undefined, null, 0, 7]) { + expect(waitpointIdFromEdgeField(edgeField("w_a", index))).toBe("w_a"); + } + }); + + it("returns undefined for a field with no separator", () => { + expect(waitpointIdFromEdgeField("nope")).toBeUndefined(); + }); + + it("splits on the last separator, tolerating a '#' inside the waitpoint id", () => { + expect(waitpointIdFromEdgeField("a#b#3")).toBe("a#b"); + }); +}); + +describe("watcherField", () => { + it("keys by run id and batch index, so one run can watch at several indexes", () => { + expect(watcherField("run_a", 2)).toBe("run_a#2"); + expect(watcherField("run_a")).toBe("run_a#"); + expect(watcherField("run_a", 0)).not.toBe(watcherField("run_a")); + }); +}); + +describe("assertSingleSlot", () => { + it("accepts keys that share one tag", () => { + const k = runBlockKeys("run_abc"); + expect(() => assertSingleSlot("runReadBlockState", [k.pend, k.done, k.edge])).not.toThrow(); + }); + + it("accepts a single tagged key", () => { + expect(() => assertSingleSlot("wpIdemReserve", [idempotencyKey("env_1", "k")])).not.toThrow(); + }); + + it("accepts an empty key list", () => { + expect(() => assertSingleSlot("noKeys", [])).not.toThrow(); + }); + + it("rejects keys from two different tags", () => { + const wp = waitpointKeys("w_a"); + const run = runBlockKeys("run_abc"); + expect(() => assertSingleSlot("bad", [wp.record, run.pend])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an untagged key", () => { + expect(() => assertSingleSlot("bad", ["wp:no-tag"])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an empty tag", () => { + expect(() => assertSingleSlot("bad", ["wp:{}"])).toThrow(WaitpointKeyTagError); + }); + + it("rejects an empty first pair, matching Redis rather than skipping to a later one", () => { + // Redis stops at the first `{`/`}` pair. An empty one means no tag at all, so it hashes + // the whole key. A regex would have found `a` here and wrongly claimed a shared slot. + expect(() => assertSingleSlot("bad", ["wp:{}{a}", "wp:{}{a}"])).toThrow(WaitpointKeyTagError); + }); + + it("takes the first pair when several are present", () => { + expect(() => assertSingleSlot("ok", ["wp:{a}{b}", "wp:{a}:w"])).not.toThrow(); + expect(() => assertSingleSlot("bad", ["wp:{a}{b}", "wp:{b}:w"])).toThrow(WaitpointKeyTagError); + }); + + it("does not degrade on a key made of many opening braces", () => { + const started = performance.now(); + expect(() => assertSingleSlot("bad", ["{".repeat(50_000)])).toThrow(WaitpointKeyTagError); + expect(performance.now() - started).toBeLessThan(1_000); + }); + + it("names the operation and the offending key in the error", () => { + const wp = waitpointKeys("w_a"); + const run = runBlockKeys("run_abc"); + try { + assertSingleSlot("myOperation", [wp.record, run.pend]); + throw new Error("should have thrown"); + } catch (error) { + expect(error).toBeInstanceOf(WaitpointKeyTagError); + expect((error as Error).message).toContain("myOperation"); + expect((error as Error).message).toContain(run.pend); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts new file mode 100644 index 00000000000..28eac087b4b --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts @@ -0,0 +1,98 @@ +/** + * Waitpoint coordination keyspace. Two hash tags, deliberately: + * + * - `wp:{waitpointId}` — the record, its status and completion envelope, plus the + * watcher hash. A waitpoint has N watcher runs, so it cannot live under any single + * run's tag. + * - `wp:run:{runId}:*` — one run's pending set, delivered set and edge set. The pending + * set's cardinality is the blocked-versus-unblocked signal, so it has to be readable + * atomically, which means one slot. + * + * Every script therefore touches exactly one tag, and assertSingleSlot enforces it on + * every invocation. A cluster would reject a cross-slot script; a single-node test server + * would not, so this assertion is the only thing standing between a cross-slot bug and + * production. + */ + +export type WaitpointKeys = { record: string; watchers: string }; +export type RunBlockKeys = { pend: string; done: string; edge: string }; + +export function waitpointKeys(waitpointId: string): WaitpointKeys { + const base = `wp:{${waitpointId}}`; + return { record: base, watchers: `${base}:w` }; +} + +export function runBlockKeys(runId: string): RunBlockKeys { + const base = `wp:run:{${runId}}`; + return { pend: `${base}:pend`, done: `${base}:done`, edge: `${base}:edge` }; +} + +export function idempotencyKey(environmentId: string, key: string): string { + return `wp:idem:{${environmentId}}:${key}`; +} + +// "#" separates the id from the index. An absent index collapses onto the empty suffix, +// which is how the partial unique index on a null batchIndex behaves; index 0 keeps its +// own field, because "0" and "" are different strings. The split back to an id below is +// taken from the LAST "#", not the first, so this stays unambiguous even if a waitpoint id +// or a run id ever contains "#" itself. +const SEPARATOR = "#"; + +export function edgeField(waitpointId: string, batchIndex?: number | null): string { + return `${waitpointId}${SEPARATOR}${batchIndex ?? ""}`; +} + +export function watcherField(runId: string, batchIndex?: number | null): string { + return `${runId}${SEPARATOR}${batchIndex ?? ""}`; +} + +// The last-"#" rule here is re-implemented as a Lua pattern in runClear (scripts.ts). This +// function has no caller besides its own test, so that test is what pins the rule as a +// specification the Lua mirrors, not just documentation of this helper. +export function waitpointIdFromEdgeField(field: string): string | undefined { + const separator = field.lastIndexOf(SEPARATOR); + return separator === -1 ? undefined : field.slice(0, separator); +} + +export class WaitpointKeyTagError extends Error { + constructor(operation: string, keys: string[], offending: string) { + super( + `Waitpoint operation ${operation} would span more than one cluster slot: ` + + `key ${JSON.stringify(offending)} does not share the tag of ${JSON.stringify(keys)}` + ); + this.name = "WaitpointKeyTagError"; + } +} + +// Redis's own keyHashSlot rule: the FIRST `{`, then the FIRST `}` after it. A missing brace +// or an empty pair means no tag, and Redis hashes the whole key. A regex would instead find +// the first NON-empty pair, disagreeing with Redis on `wp:{}{a}`. +function hashTag(key: string): string | undefined { + const open = key.indexOf("{"); + if (open === -1) return undefined; + + const close = key.indexOf("}", open + 1); + if (close === -1 || close === open + 1) return undefined; + + return key.slice(open + 1, close); +} + +/** + * Throw unless every key carries the same non-empty hash tag. Called on every script + * invocation, because the keys embed ids and are only known at call time. + */ +export function assertSingleSlot(operation: string, keys: string[]): void { + let tag: string | undefined; + + for (const key of keys) { + const found = hashTag(key); + if (!found) { + throw new WaitpointKeyTagError(operation, keys, key); + } + if (tag === undefined) { + tag = found; + } else if (found !== tag) { + throw new WaitpointKeyTagError(operation, keys, key); + } + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts new file mode 100644 index 00000000000..820b4145f86 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts @@ -0,0 +1,388 @@ +import type { Callback, Redis, Result } from "@internal/redis"; + +/** + * Lua for the waitpoint coordination protocol. Three rules hold throughout: + * + * 1. Every key a script touches is declared in KEYS. No script builds a key name inside + * Lua. ioredis prefixes only the KEYS array, so a key minted in Lua would be + * unprefixed while the client wrote a prefixed one — and a script with a single + * declared key gives the caller's single-slot assertion nothing to compare. + * 2. Lua never parses JSON. Each script branches only on a short status string and moves + * opaque blobs, so every encoding decision stays in TypeScript. + * 3. A missing HGET returns Lua `false`, not `nil` — measured directly against a live + * Redis: `EVAL "return {'a', false, 'c'}"` and a table holding a missing-field HGET + * result both come back as 3 elements; only `EVAL "return {'a', nil, 'c'}"` comes back + * as 1. A `false` element converts to a reply-array null and does NOT shorten anything + * after it — only a genuine Lua nil truncates. Every returned slot is still coerced + * with `or ''` regardless, not to prevent truncation, but so an absent value arrives + * as `''` rather than `null`, giving the TypeScript one shape to decode instead of + * two. + * + * STORED_COMPLETED is the value written into the record's `status` field and is + * UPPERCASE. The outcome tokens below are lowercase and are a separate vocabulary: they + * name what a script DID, not what a record IS. Sharing one constant between the two + * makes an already-completed record invisible to every script. + */ + +const STORED_COMPLETED = "COMPLETED"; + +const MISSING = "missing"; +const CREATED = "created"; +const EXISTS = "exists"; +const REGISTERED = "registered"; +const DID_COMPLETE = "completed"; +const ALREADY = "already"; +const RESERVED = "reserved"; +const CLEARED = "cleared"; +const DRAINED = "drained"; +const DISCARDED = "discarded"; + +export function registerWaitpointCommands(redis: Redis): void { + // KEYS: record. ARGV: recordJson, status ('PENDING'|'COMPLETED'), completionJson (''). + redis.defineCommand("wpCreateIfAbsent", { + numberOfKeys: 1, + lua: ` + local record = KEYS[1] + + -- EXISTS-then-HSET inside one script, rather than a field-by-field HSETNX: the + -- record and its status must appear together or not at all. + if redis.call('EXISTS', record) == 1 then + local vals = redis.call('HMGET', record, 'r', 'status', 'c') + return { '${EXISTS}', vals[1] or '', vals[2] or '', vals[3] or '' } + end + + redis.call('HSET', record, 'r', ARGV[1], 'status', ARGV[2]) + if ARGV[3] ~= '' then + redis.call('HSET', record, 'c', ARGV[3]) + end + + return { '${CREATED}' } + `, + }); + + // KEYS: record, watchers. ARGV: watcherField, watcherJson. + redis.defineCommand("wpRegisterOrReport", { + numberOfKeys: 2, + lua: ` + local record, watchers = KEYS[1], KEYS[2] + + -- A missing waitpoint is never a silent no-op: the caller throws. Defaulting to + -- "not blocked" here would resume a run whose waitpoint never completed. + if redis.call('EXISTS', record) == 0 then + return { '${MISSING}' } + end + + if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then + return { '${DID_COMPLETE}', redis.call('HGET', record, 'c') or '' } + end + + -- The watcher lands before any flip can read the watcher hash, because this script + -- and wpComplete are both atomic on this same shard. So a register either appears + -- in the flip's watcher list, or it observes COMPLETED above. + -- + -- HSETNX: the first registration wins, mirroring the edge's ON CONFLICT DO NOTHING. + redis.call('HSETNX', watchers, ARGV[1], ARGV[2]) + return { '${REGISTERED}' } + `, + }); + + // KEYS: record, watchers. ARGV: completionJson. + redis.defineCommand("wpComplete", { + numberOfKeys: 2, + lua: ` + local record, watchers = KEYS[1], KEYS[2] + + if redis.call('EXISTS', record) == 0 then + return { '${MISSING}' } + end + + local outcome = '${DID_COMPLETE}' + if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then + -- Double completion is not an error, and the FIRST completion wins. This is the + -- guard a conditional UPDATE ... WHERE status = 'PENDING' used to provide. + outcome = '${ALREADY}' + else + redis.call('HSET', record, 'status', '${STORED_COMPLETED}', 'c', ARGV[1]) + end + + -- Returning the watchers here is what removes the reverse fan-out query. The + -- envelope comes back too, because delivery runs on each watcher's own shard and + -- cannot read this key. + local out = { outcome, redis.call('HGET', record, 'c') or '' } + local entries = redis.call('HVALS', watchers) + for i = 1, #entries do + out[#out + 1] = entries[i] + end + + return out + `, + }); + + // KEYS: idempotency key. ARGV: waitpointId, expiresAtMs ('' for no expiry). + redis.defineCommand("wpIdemReserve", { + numberOfKeys: 1, + lua: ` + local key = KEYS[1] + + -- Guard before the SET: a non-numeric expiry must not land a reservation that can + -- never expire because PEXPIREAT then errors out after the write already happened. + if ARGV[2] ~= '' and tonumber(ARGV[2]) == nil then + return redis.error_reply('wpIdemReserve: ARGV[2] must be numeric or empty') + end + + -- SET NX returns a status reply on success and false on conflict. + if redis.call('SET', key, ARGV[1], 'NX') then + -- Expiry only when the caller has one. A reservation with no expiry is the common + -- case and must never grow one here. + if ARGV[2] ~= '' then + redis.call('PEXPIREAT', key, tonumber(ARGV[2])) + end + return { '${RESERVED}', ARGV[1] } + end + + return { '${EXISTS}', redis.call('GET', key) or '' } + `, + }); + + // KEYS: record, watchers. No ARGV. Discards a losing reservation's orphan record. + redis.defineCommand("wpDiscard", { + numberOfKeys: 2, + lua: ` + redis.call('DEL', KEYS[1], KEYS[2]) + return { '${DISCARDED}' } + `, + }); + + // KEYS: pend, done, edge. + // ARGV: n, then n groups of 5 — waitpointId, edgeField, edgeJson, reportedFlag + // ('1'|'0'), reportedJson (''). reportedFlag, not the emptiness of reportedJson, is what + // decides the branch: a waitpoint can be reported COMPLETED with no completion envelope + // (see the FINISHED-healing path), and that case must still take the reported branch — + // flag '1', reportedJson '' — or the run would block forever on something already done. + redis.defineCommand("runAbsorbBlockers", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + local n = tonumber(ARGV[1]) + + -- Guard before any write: a wrong n must not half-apply the script. HDEL/HSETNX below + -- are irreversible mid-script, and Redis does not roll back a script that errors. + if #ARGV ~= 1 + n * 5 then + return redis.error_reply('runAbsorbBlockers: arity mismatch') + end + + -- seenDelivered makes the delivered-pair output DISTINCT BY ID: two edges for one + -- waitpoint must contribute one pair, not two. + local requestedIds = {} + local seenDelivered = {} + local out = { '0', '0' } + + for i = 0, n - 1 do + local id = ARGV[2 + i * 5] + local field = ARGV[3 + i * 5] + local edgeJson = ARGV[4 + i * 5] + local reportedFlag = ARGV[5 + i * 5] + local reported = ARGV[6 + i * 5] + + -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not + -- overwrite the first attempt's metadata. + redis.call('HSETNX', edge, field, edgeJson) + requestedIds[id] = true + + if reportedFlag == '1' then + -- Already COMPLETED when the watcher registered. It never becomes pending, even + -- when reported ('' here) carries no envelope. + redis.call('HSET', done, id, reported) + redis.call('SREM', pend, id) + if not seenDelivered[id] then + seenDelivered[id] = true + out[#out + 1] = id + out[#out + 1] = reported + end + else + -- Check the delivered set FIRST. A completion that landed between register and + -- absorb has already delivered here, and that delivery wins. + local delivered = redis.call('HGET', done, id) + if delivered then + if not seenDelivered[id] then + seenDelivered[id] = true + out[#out + 1] = id + out[#out + 1] = delivered + end + else + redis.call('SADD', pend, id) + end + end + end + + -- Computed AFTER every write in this batch, as the count of distinct requested ids + -- with no entry in done. Counting incrementally during the loop is order-dependent: + -- a later group's completion for an id already counted as pending would leave the + -- count stale, reporting a waitpoint as both pending and delivered. + local pendingOfRequested = 0 + for id in pairs(requestedIds) do + if redis.call('HEXISTS', done, id) == 0 then + pendingOfRequested = pendingOfRequested + 1 + end + end + + out[1] = tostring(pendingOfRequested) + out[2] = tostring(redis.call('SCARD', pend)) + return out + `, + }); + + // KEYS: pend, done. ARGV: waitpointId, completionJson. + redis.defineCommand("runDeliverCompletion", { + numberOfKeys: 2, + lua: ` + local pend, done = KEYS[1], KEYS[2] + + redis.call('HSET', done, ARGV[1], ARGV[2]) + redis.call('SREM', pend, ARGV[1]) + + -- The caller treats this as a wakeup trigger, not as the resume decision: the + -- resume is decided under the run lock, and this count covers store-resident + -- blockers only. + return { tostring(redis.call('SCARD', pend)) } + `, + }); + + // KEYS: pend, done, edge. + redis.defineCommand("runReadBlockState", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + + local pendIds = redis.call('SMEMBERS', pend) + -- HKEYS, never HGETALL: the delivered set's values are completion envelopes with + -- inline outputs, and materializing those inside a single-threaded script would + -- block the shard. + local doneIds = redis.call('HKEYS', done) + local edges = redis.call('HGETALL', edge) + + local out = { tostring(#pendIds), tostring(#doneIds), tostring(#edges) } + for i = 1, #pendIds do out[#out + 1] = pendIds[i] end + for i = 1, #doneIds do out[#out + 1] = doneIds[i] end + for i = 1, #edges do out[#out + 1] = edges[i] end + return out + `, + }); + + // KEYS: pend, done, edge. ARGV: n, then n edge fields. n = 0 clears everything. + redis.defineCommand("runClear", { + numberOfKeys: 3, + lua: ` + local pend, done, edge = KEYS[1], KEYS[2], KEYS[3] + local n = tonumber(ARGV[1]) + + -- Guard before any write, same reasoning as runAbsorbBlockers. + if #ARGV ~= 1 + n then + return redis.error_reply('runClear: arity mismatch') + end + + if n == 0 then + redis.call('DEL', pend, done, edge) + return { '${CLEARED}' } + end + + for i = 1, n do + redis.call('HDEL', edge, ARGV[1 + i]) + end + + -- Reconcile rather than delete by name. The edge set is the authority: after the + -- drain, pend and done may only hold ids that some surviving edge still references. + -- + -- Two reasons this is a superset of "remove the drained ids". First, one waitpoint + -- can hold several edges at different batch indexes, so a drained field must not + -- evict a delivery another edge still needs. Second, runDeliverCompletion writes + -- done[id] unconditionally, so a crash between register and absorb can leave a + -- delivered entry with no edge at all, which no name-derived drain could reach. + local remaining = {} + local fields = redis.call('HKEYS', edge) + for i = 1, #fields do + local sep = string.find(fields[i], '#[^#]*$') + if sep then + remaining[string.sub(fields[i], 1, sep - 1)] = true + end + end + + local doneIds = redis.call('HKEYS', done) + for i = 1, #doneIds do + if not remaining[doneIds[i]] then + redis.call('HDEL', done, doneIds[i]) + end + end + + local pendIds = redis.call('SMEMBERS', pend) + for i = 1, #pendIds do + if not remaining[pendIds[i]] then + redis.call('SREM', pend, pendIds[i]) + end + end + + return { '${DRAINED}' } + `, + }); +} + +declare module "@internal/redis" { + interface RedisCommander { + wpCreateIfAbsent( + recordKey: string, + recordJson: string, + status: string, + completionJson: string, + callback?: Callback + ): Result; + wpRegisterOrReport( + recordKey: string, + watchersKey: string, + watcherField: string, + watcherJson: string, + callback?: Callback + ): Result; + wpComplete( + recordKey: string, + watchersKey: string, + completionJson: string, + callback?: Callback + ): Result; + wpIdemReserve( + key: string, + waitpointId: string, + expiresAtMs: string, + callback?: Callback + ): Result; + wpDiscard( + recordKey: string, + watchersKey: string, + callback?: Callback + ): Result; + runAbsorbBlockers( + pendKey: string, + doneKey: string, + edgeKey: string, + ...args: Array> + ): Result; + runDeliverCompletion( + pendKey: string, + doneKey: string, + waitpointId: string, + completionJson: string, + callback?: Callback + ): Result; + runReadBlockState( + pendKey: string, + doneKey: string, + edgeKey: string, + callback?: Callback + ): Result; + runClear( + pendKey: string, + doneKey: string, + edgeKey: string, + ...args: Array> + ): Result; + } +} diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts new file mode 100644 index 00000000000..f0e9c0c297d --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts @@ -0,0 +1,1837 @@ +// Redis-only suite: the coordinator holds no Prisma reference, so no Postgres container +// is needed. redisTest FLUSHALLs before every test, so ids may be reused across describes. +import { createRedisClient, type RedisOptions } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { + deriveWaitpointIdFromAnchor, + generateRunOpsId, + generateWaitpointId, +} from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { + edgeField, + idempotencyKey, + runBlockKeys, + watcherField, + WaitpointKeyTagError, +} from "./keys.js"; +import { registerWaitpointCommands } from "./scripts.js"; +import { + WaitpointNotFoundError, + WaitpointStoreCoordinator, + type BlockEdge, + type WaitpointCompletion, + type WaitpointRecordInput, + type WatcherEntry, +} from "./storeCoordinator.js"; + +const ENV_ID = "env_1"; +const PROJECT_ID = "proj_1"; +const NOW = "2026-08-21T12:00:00.000Z"; + +function coordinator(redisOptions: RedisOptions) { + return new WaitpointStoreCoordinator({ redisOptions }); +} + +function record(id: string, overrides: Partial = {}): WaitpointRecordInput { + return { + id, + friendlyId: `waitpoint_${id}`, + type: "MANUAL", + environmentId: ENV_ID, + projectId: PROJECT_ID, + createdAt: NOW, + updatedAt: NOW, + userProvidedIdempotencyKey: false, + tags: [], + ...overrides, + }; +} + +function completion(overrides: Partial = {}): WaitpointCompletion { + return { + completedAt: NOW, + outputType: "application/json", + outputIsError: false, + output: { inline: '{"ok":true}' }, + ...overrides, + }; +} + +describe("createIfAbsent", () => { + redisTest("creates a PENDING record and reports created", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + expect(result.outcome).toBe("created"); + } finally { + await store.quit(); + } + }); + + redisTest("returns the existing record on a second call", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const second = await store.createIfAbsent({ + record: record("w_a", { friendlyId: "waitpoint_DIFFERENT" }), + status: "PENDING", + }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + // The first write wins: a retry must not overwrite the stored record. + expect(second.record.friendlyId).toBe("waitpoint_w_a"); + expect(second.status).toBe("PENDING"); + expect(second.completion).toBeUndefined(); + } finally { + await store.quit(); + } + }); + + redisTest("preserves every record field through a round trip", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const full = record("w_a", { + type: "RUN", + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: NOW, + completedAfter: NOW, + completedByTaskRunId: "run_child", + completedByBatchId: "batch_1", + tags: ["one", "two"], + }); + + await store.createIfAbsent({ record: full, status: "PENDING" }); + const read = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(read.outcome).toBe("exists"); + if (read.outcome !== "exists") throw new Error("unreachable"); + // Every field the frozen return shapes need must survive the blob round trip. + expect(read.record).toEqual(full); + } finally { + await store.quit(); + } + }); + + redisTest( + "can create an already-COMPLETED record with no completion envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // This is the shape that catches a status-casing mismatch: the record is stored + // COMPLETED, and a register must see it as completed rather than pending. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const reported = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(reported.outcome).toBe("completed"); + if (reported.outcome !== "completed") throw new Error("unreachable"); + expect(reported.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "can create an already-COMPLETED record with a completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a", { type: "RUN" }), + status: "COMPLETED", + completion: completion(), + }); + + const reported = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(reported.outcome).toBe("completed"); + if (reported.outcome !== "completed") throw new Error("unreachable"); + expect(reported.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "reads a COMPLETED record back through createIfAbsent, with an envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a"), + status: "COMPLETED", + completion: completion(), + }); + + const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + expect(second.status).toBe("COMPLETED"); + expect(second.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "reads a COMPLETED record back through createIfAbsent, with no envelope", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + expect(second.outcome).toBe("exists"); + if (second.outcome !== "exists") throw new Error("unreachable"); + expect(second.status).toBe("COMPLETED"); + expect(second.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerOrReport", () => { + redisTest("registers a watcher against a PENDING waitpoint", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const result = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + expect(result.outcome).toBe("registered"); + } finally { + await store.quit(); + } + }); + + redisTest("reports the completion inline for a COMPLETED waitpoint", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + const result = await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + createdAt: NOW, + }); + + expect(result.outcome).toBe("completed"); + if (result.outcome !== "completed") throw new Error("unreachable"); + expect(result.completion?.output).toEqual({ inline: '{"ok":true}' }); + } finally { + await store.quit(); + } + }); + + redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.registerOrReport({ waitpointId: "w_missing", runId: "run_1", createdAt: NOW }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest("keeps one watcher entry per batch index", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + batchIndex: 0, + createdAt: NOW, + }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + batchIndex: 2, + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers).toHaveLength(2); + expect(completed.watchers.map((w) => w.batchIndex).sort((a, b) => a! - b!)).toEqual([0, 2]); + } finally { + await store.quit(); + } + }); + + redisTest("carries spanIdToComplete through to the watcher entry", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_abc", + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers[0]!.spanIdToComplete).toBe("span_abc"); + expect(completed.watchers[0]!.runId).toBe("run_1"); + expect(completed.watchers[0]!.createdAt).toBe(NOW); + } finally { + await store.quit(); + } + }); + + redisTest("keeps the first registration's watcher on a re-register", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_first", + createdAt: NOW, + }); + // Same run, same (absent) batch index, so the watcher field collides. HSETNX must + // not let this second registration overwrite the first one's span. + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_second", + createdAt: NOW, + }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers).toHaveLength(1); + expect(completed.watchers[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + }); +}); + +describe("complete", () => { + redisTest("flips PENDING to COMPLETED and returns the watchers", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW }); + + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + + expect(result.outcome).toBe("completed"); + expect(result.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent and returns the watchers again", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + + const first = await store.complete({ waitpointId: "w_a", completion: completion() }); + const second = await store.complete({ + waitpointId: "w_a", + completion: completion({ output: { inline: '{"second":true}' } }), + }); + + expect(first.outcome).toBe("completed"); + expect(second.outcome).toBe("already"); + // The FIRST completion wins, matching the guard on status = PENDING. + expect(second.completion?.output).toEqual({ inline: '{"ok":true}' }); + expect(second.watchers.map((w) => w.runId)).toEqual(["run_1"]); + } finally { + await store.quit(); + } + }); + + redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.complete({ waitpointId: "w_missing", completion: completion() }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty watcher list when nobody is blocked", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(result.watchers).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "keeps the watcher list intact when the completion field is absent on an already-completed record", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + // registerOrReport never lets a watcher land once status is COMPLETED, so this + // shape is forced by hand: it pins that an absent 'c' field decodes to an + // undefined completion without disturbing the watchers that follow it in the + // reply array. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + const watcher: WatcherEntry = { runId: "run_1", createdAt: NOW }; + await probe.hset("wp:{w_a}:w", watcherField("run_1"), JSON.stringify(watcher)); + + const result = await store.complete({ waitpointId: "w_a", completion: completion() }); + + expect(result.outcome).toBe("already"); + expect(result.completion).toBeUndefined(); + expect(result.watchers).toHaveLength(1); + expect(result.watchers[0]!.runId).toBe("run_1"); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest("sets no TTL on the record or the watcher key", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + // -1 means the key exists with no expiry. Anything >= 0 breaks the retention rule. + expect(await probe.pttl("wp:{w_a}")).toBe(-1); + expect(await probe.pttl("wp:{w_a}:w")).toBe(-1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); +}); + +// A coordinator method now drives each of these scripts, but this block stays: it is the +// only place asserting the RAW reply shape, so a Lua/TypeScript framing change made on +// both sides at once would still fail here even though every class-level test passed. +describe("reply framing (direct Lua — pins the wire shape the coordinator decodes)", () => { + const envelope = JSON.stringify(completion()); + + redisTest( + "does not double-count a waitpoint reported pending then delivered in the same batch", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const fieldA = edgeField("w_solo", 0); + const fieldB = edgeField("w_solo", 1); + + // Group 0 arrives unreported (still pending); group 1 for the SAME waitpoint + // arrives already reported. This is the straddle that broke pendingOfRequested. + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + fieldA, + "{}", + "0", + "", + "w_solo", + fieldB, + "{}", + "1", + envelope + ); + + expect(reply).toEqual(["0", "0", "w_solo", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "produces the identical result when the same two groups arrive in reverse order", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const fieldA = edgeField("w_solo", 0); + const fieldB = edgeField("w_solo", 1); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + fieldB, + "{}", + "1", + envelope, + "w_solo", + fieldA, + "{}", + "0", + "" + ); + + expect(reply).toEqual(["0", "0", "w_solo", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest("counts two distinct unreported ids as fully pending", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_b", + edgeField("w_b", 0), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["2", "2"]); + expect(await client.scard(keys.pend)).toBe(2); + } finally { + client.disconnect(); + } + }); + + redisTest( + "counts one reported and one unreported id as one pending, one delivered", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_b", + edgeField("w_b", 0), + "{}", + "1", + envelope + ); + + expect(reply).toEqual(["1", "1", "w_b", envelope]); + expect(await client.scard(keys.pend)).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "reported flag '1' with an empty envelope still delivers, not pends", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + // The bug this task fixed: COMPLETED-with-no-envelope must take the reported + // branch on the flag alone, not on the envelope being non-empty. + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "1", + "w_a", + edgeField("w_a", 0), + "{}", + "1", + "" + ); + + expect(reply).toEqual(["0", "0", "w_a", ""]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "counts the same unreported id passed twice as one pending, not two", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "", + "w_a", + edgeField("w_a", 1), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["1", "1"]); + expect(await client.scard(keys.pend)).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "counts an id already in done, passed unreported, as delivered rather than pending", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + // A completion that landed between register and absorb — the delivered set + // already has this id before the absorb call ever sees it. + await client.hset(keys.done, "w_a", envelope); + + const reply = await client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "1", + "w_a", + edgeField("w_a", 0), + "{}", + "0", + "" + ); + + expect(reply).toEqual(["0", "0", "w_a", envelope]); + expect(await client.scard(keys.pend)).toBe(0); + } finally { + client.disconnect(); + } + } + ); + + redisTest("rejects an arity mismatch before writing anything", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const field = edgeField("w_solo", 0); + + // n says 2 groups (1 + 2 * 5 = 11 ARGV entries expected) but only one group (5 + // ARGV entries) is supplied. + await expect( + client.runAbsorbBlockers( + keys.pend, + keys.done, + keys.edge, + "2", + "w_solo", + field, + "{}", + "0", + "" + ) + ).rejects.toThrow(); + + expect(await client.exists(keys.pend)).toBe(0); + expect(await client.exists(keys.done)).toBe(0); + expect(await client.exists(keys.edge)).toBe(0); + } finally { + client.disconnect(); + } + }); + + redisTest( + "runClear rejects an arity mismatch before writing anything", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const keys = runBlockKeys("run_1"); + const field = edgeField("w_solo", 0); + await client.hset(keys.edge, field, "{}"); + await client.sadd(keys.pend, "w_solo"); + + // n says 2 fields but only one field is supplied. + await expect( + client.runClear(keys.pend, keys.done, keys.edge, "2", field) + ).rejects.toThrow(); + + expect(await client.hexists(keys.edge, field)).toBe(1); + expect(await client.sismember(keys.pend, "w_solo")).toBe(1); + } finally { + client.disconnect(); + } + } + ); + + redisTest( + "wpIdemReserve rejects a non-numeric expiry and does not create the reservation", + async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + registerWaitpointCommands(client); + try { + const key = idempotencyKey(ENV_ID, "key-1"); + + await expect(client.wpIdemReserve(key, "w_a", "not-a-number")).rejects.toThrow(); + + expect(await client.exists(key)).toBe(0); + } finally { + client.disconnect(); + } + } + ); +}); + +describe("createWithIdempotencyKey", () => { + // Real minted ids. The method rejects anything but a standalone DATETIME/MANUAL id, because + // its loser-discard is only safe for an id that was never handed out. + const idA = generateWaitpointId("MANUAL"); + const idB = generateWaitpointId("MANUAL"); + redisTest("creates the waitpoint and wins the reservation", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(result).toEqual({ waitpointId: idA, created: true }); + } finally { + await store.quit(); + } + }); + + redisTest("returns the winner's id and deletes the loser", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + const second = await store.createWithIdempotencyKey({ + record: record(idB, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(second).toEqual({ waitpointId: idA, created: false }); + // The loser cleans up after itself: nothing ever referenced its id. + expect(await probe.exists(`wp:{${idB}}`)).toBe(0); + expect(await probe.exists(`wp:{${idA}}`)).toBe(1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest( + "the original creator's own retry does not discard its own record", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + const withKey = record(idA, { + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + }); + + const first = await store.createWithIdempotencyKey({ + record: withKey, + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + expect(first).toEqual({ waitpointId: idA, created: true }); + + // The SAME caller, retrying with the SAME record id and the SAME key — not a + // different id racing for the same reservation. + const retry = await store.createWithIdempotencyKey({ + record: withKey, + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + expect(retry).toEqual({ waitpointId: idA, created: false }); + // The record must survive: a wrongly-discarded record would delete this too. + expect(await probe.exists(`wp:{${idA}}`)).toBe(1); + + // The real proof: something usable is still there for every later caller that + // blocks on this id. + const registered = await store.registerOrReport({ + waitpointId: idA, + runId: "run_1", + createdAt: NOW, + }); + expect(registered.outcome).toBe("registered"); + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest("sets no expiry when the record carries none", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + // The common case. An expiry appearing here would be a retention rule violation. + expect(await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`)).toBe(-1); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest("sets the expiry the record carries", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { + idempotencyKey: "key-1", + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }); + + const ttl = await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`); + // Wide band, deliberately: the deadline is computed from the test process's clock + // and applied as an absolute PEXPIREAT, while PTTL is computed against the Redis + // server's own clock. A few ms of disagreement between those two clocks is normal + // and shows up as overshoot on this read, not as a bug in the reservation. The + // band still catches every failure worth catching — wrong units, no expiry + // applied, a negative TTL — without re-asserting that two independent clocks + // agree to the millisecond. + expect(ttl).toBeGreaterThan(55_000); + expect(ttl).toBeLessThanOrEqual(65_000); + } finally { + probe.disconnect(); + await store.quit(); + } + }); + + redisTest("scopes reservations by environment", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createWithIdempotencyKey({ + record: record(idA, { idempotencyKey: "key-1" }), + environmentId: "env_1", + idempotencyKey: "key-1", + }); + + const other = await store.createWithIdempotencyKey({ + record: record(idB, { idempotencyKey: "key-1", environmentId: "env_2" }), + environmentId: "env_2", + idempotencyKey: "key-1", + }); + + expect(other).toEqual({ waitpointId: idB, created: true }); + } finally { + await store.quit(); + } + }); +}); + +redisTest( + "rejects a derived RUN id, whose loser-discard would be unsafe", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // A derived id is recomputable from its anchor, so another caller can register a + // watcher on it. Discarding one could delete a record already in use. + const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsId()}`, "RUN")!; + await expect( + store.createWithIdempotencyKey({ + record: record(derived, { type: "RUN", idempotencyKey: "key-1" }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }) + ).rejects.toThrow(/freshly minted DATETIME or MANUAL/); + } finally { + await store.quit(); + } + } +); + +describe("the single-slot guard", () => { + redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // Reaches the same wrapper every operation goes through, so this proves the guard + // is live at the call path and not only in the pure unit test. + expect(() => + store.assertKeysForTest("wpComplete", ["wp:{w_a}", "wp:run:{run_1}:pend"]) + ).toThrow(WaitpointKeyTagError); + } finally { + await store.quit(); + } + }); +}); + +const RUN_ID = "run_1"; + +function edge(waitpointId: string, overrides: Partial = {}): BlockEdge { + return { waitpointId, createdAt: NOW, type: "MANUAL", ...overrides }; +} + +describe("absorbBlockers", () => { + redisTest("counts pending blockers and reports the store total", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a"), edge("w_b")], + }); + + expect(result.pendingOfRequested).toBe(2); + expect(result.storePendingTotal).toBe(2); + expect(result.alreadyDelivered).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "counts a repeated waitpoint id once, matching a count over distinct rows", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 2 })], + }); + + // The count this replaces was a COUNT(*) over waitpoint rows, so two edges for + // one waitpoint contributed one. Both numbers must say 1, not 2. + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges).toHaveLength(2); + expect(state.edges.map((e) => e.batchIndex).sort()).toEqual([0, 2]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "does not add a reported-complete blocker to the pending set", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: completion() } }), edge("w_b")], + }); + + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a later absorb reads back the stored envelope, not a bare flag", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const envelope = completion({ output: { inline: '{"first":true}' } }); + + // Reported once, with an envelope — this write is what's under test. + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: envelope } })], + }); + + // Same waitpoint id, arriving unreported this time: takes the "read `done` back" + // path, exposing whatever the first call actually stored under that id. + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(second.alreadyDelivered).toHaveLength(1); + expect(second.alreadyDelivered[0]!.completion).toEqual(envelope); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a later absorb for a no-envelope delivery reads back no completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: {} })], + }); + + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(second.alreadyDelivered).toHaveLength(1); + expect(second.alreadyDelivered[0]!.completion).toBeUndefined(); + } finally { + await store.quit(); + } + } + ); + + redisTest("reports a repeated already-delivered id once", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [ + edge("w_a", { batchIndex: 0, reported: { completion: completion() } }), + edge("w_a", { batchIndex: 1, reported: { completion: completion() } }), + ], + }); + + expect(result.alreadyDelivered).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest("lets a delivery that raced ahead of the absorb win", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + const result = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent when run twice", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + const first = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(first.storePendingTotal).toBe(1); + expect(second.storePendingTotal).toBe(1); + expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest("keeps the first edge's metadata on a retry", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { spanIdToComplete: "span_first" })], + }); + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { spanIdToComplete: "span_second" })], + }); + + expect((await store.readBlockState(RUN_ID)).edges[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + }); + + redisTest("reports the run's real total for an empty edge list", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + + const result = await store.absorbBlockers({ runId: RUN_ID, edges: [] }); + + // pendingOfRequested is 0 because nothing was requested. storePendingTotal is the + // run's whole store-resident set, which is NOT empty. + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest( + "reports a smaller pendingOfRequested than storePendingTotal when an unrelated blocker is already pending", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // w_x is a live blocker from an earlier absorb, unrelated to this call's request. + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_x")] }); + + const result = await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { reported: { completion: completion() } })], + }); + + // Nothing THIS call requested is pending (w_a arrived already delivered), but the + // run's whole store-resident set still holds w_x — a divergence for a different + // reason than an empty request list, so a reply[0]/reply[1] swap or a + // re-derived-in-TypeScript pendingOfRequested would both be caught here too. + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(1); + } finally { + await store.quit(); + } + } + ); + + redisTest("sets no TTL on any run key", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + // -1 is "exists, no expiry"; -2 is "no key". Neither is a TTL. `pend` is emptied by + // the delivery, and Redis deletes an empty set, so -2 is expected there. + for (const key of [ + `wp:run:{${RUN_ID}}:pend`, + `wp:run:{${RUN_ID}}:done`, + `wp:run:{${RUN_ID}}:edge`, + ]) { + expect(await probe.pttl(key)).toBeLessThan(0); + } + } finally { + probe.disconnect(); + await store.quit(); + } + }); +}); + +describe("deliverCompletion", () => { + redisTest("removes the blocker and returns the new store total", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + expect( + ( + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }) + ).storePendingTotal + ).toBe(1); + + expect( + ( + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_b", + completion: completion(), + }) + ).storePendingTotal + ).toBe(0); + } finally { + await store.quit(); + } + }); + + redisTest("is idempotent", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + const again = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + expect(again.storePendingTotal).toBe(0); + } finally { + await store.quit(); + } + }); +}); + +describe("readBlockState", () => { + redisTest( + "returns the pending ids, the delivered ids and the edges", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [ + edge("w_a", { batchIndex: 0, completedAfter: NOW, type: "DATETIME" }), + edge("w_b"), + ], + }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + const state = await store.readBlockState(RUN_ID); + + expect(state.pendingIds).toEqual(["w_b"]); + expect(state.deliveredIds).toEqual(["w_a"]); + expect(state.edges).toHaveLength(2); + + const datetime = state.edges.find((e) => e.waitpointId === "w_a"); + // type and completedAfter must ride the edge: a frozen return type needs them, and + // they live on the waitpoint's own shard, which this read cannot touch. + expect(datetime?.type).toBe("DATETIME"); + expect(datetime?.completedAfter).toBe(NOW); + expect(datetime?.edgeId).toBe("w_a#0"); + } finally { + await store.quit(); + } + } + ); + + redisTest("returns empty collections for a run with no blockers", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + expect(await store.readBlockState("run_unknown")).toEqual({ + pendingIds: [], + deliveredIds: [], + edges: [], + }); + } finally { + await store.quit(); + } + }); +}); + +describe("clearBlockState", () => { + redisTest("drains the named edges and reconciles", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#"] })).outcome).toBe( + "drained" + ); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_b"]); + expect(state.deliveredIds).toEqual([]); + expect(state.pendingIds).toEqual(["w_b"]); + } finally { + await store.quit(); + } + }); + + redisTest( + "keeps a waitpoint's delivery while another edge for it survives", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 1 })], + }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completion(), + }); + + await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#0"] }); + + // One edge remains, so the delivery must remain too — dropping it would make the + // surviving edge look undelivered. + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.edgeId)).toEqual(["w_a#1"]); + expect(state.deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("reaps a delivered entry that no edge references", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // The register-before-absorb window: a delivery can land for a waitpoint whose edge + // was never written. A name-derived drain could never reach it. + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_kept")] }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_orphan", + completion: completion(), + }); + + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_orphan"]); + + await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_nothing#"] }); + + const state = await store.readBlockState(RUN_ID); + expect(state.deliveredIds).toEqual([]); + expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_kept"]); + } finally { + await store.quit(); + } + }); + + redisTest("clears everything when no edge ids are given", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + expect((await store.clearBlockState({ runId: RUN_ID })).outcome).toBe("cleared"); + expect(await store.readBlockState(RUN_ID)).toEqual({ + pendingIds: [], + deliveredIds: [], + edges: [], + }); + } finally { + await store.quit(); + } + }); + + redisTest( + "is a no-op for an explicitly empty edge id list, unlike an omitted one", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] }); + + // Omitting edgeIds reaches the Lua's n === 0 branch and clears everything (proven + // above). A caller-computed EMPTY array must not collapse onto that: it means + // "nothing to drain", not "clear the run". + expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: [] })).outcome).toBe("noop"); + + const state = await store.readBlockState(RUN_ID); + expect(state.edges.map((e) => e.waitpointId).sort()).toEqual(["w_a", "w_b"]); + expect(state.pendingIds.sort()).toEqual(["w_a", "w_b"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerBlocks: a COMPLETED waitpoint with no envelope never blocks (regression)", () => { + redisTest( + "created COMPLETED with no envelope: registerBlocks does not block the run", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + // No `completion` at all — the FINISHED-healing shape from Task 4's "can create an + // already-COMPLETED record with no completion envelope" test. + await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + // The whole point: no fabricated envelope, and the delivery is real on the run + // shard, not just absent from pending. + expect(result.alreadyDelivered[0]!.completion).toBeUndefined(); + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "created COMPLETED with an envelope: behaves identically with respect to blocking", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_a"), + status: "COMPLETED", + completion: completion(), + }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("registerBlocks: the two orderings", () => { + redisTest("block first, then complete: the run blocks, then wakes", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + const blocked = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + expect(blocked.pendingOfRequested).toBe(1); + expect(blocked.storePendingTotal).toBe(1); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]); + + const delivered = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completed.completion!, + }); + expect(delivered.storePendingTotal).toBe(0); + } finally { + await store.quit(); + } + }); + + redisTest("complete first, then block: the run never goes pending", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.complete({ waitpointId: "w_a", completion: completion() }); + + const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(result.pendingOfRequested).toBe(0); + expect(result.storePendingTotal).toBe(0); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.deliveredIds).toEqual(["w_a"]); + } finally { + await store.quit(); + } + }); + + redisTest("throws when a blocking waitpoint does not exist", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await expect( + store.registerBlocks({ runId: RUN_ID, edges: [edge("w_missing")] }) + ).rejects.toThrow(WaitpointNotFoundError); + } finally { + await store.quit(); + } + }); + + redisTest( + "a throw mid-loop leaves the earlier watcher registered, and that residue is safe", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_ok"), status: "PENDING" }); + + await expect( + store.registerBlocks({ runId: RUN_ID, edges: [edge("w_ok"), edge("w_missing")] }) + ).rejects.toThrow(WaitpointNotFoundError); + + // registerBlocks throws before absorbBlockers ever runs, so the run's own shard + // is untouched. + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.edges).toEqual([]); + + // But w_ok's watcher WAS registered on w_ok's own shard before the throw. + const completed = await store.complete({ waitpointId: "w_ok", completion: completion() }); + expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]); + + // Delivering it writes a `done` entry for a run that was never blocked on it — + // inert residue, not a false resume: no edge ever named it, and clearBlockState's + // reconcile would drop it the moment this run's block state is next drained. + const delivered = await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_ok", + completion: completed.completion!, + }); + expect(delivered.storePendingTotal).toBe(0); + expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_ok"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("is idempotent when run twice", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + + const first = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + const second = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + expect(first.storePendingTotal).toBe(1); + expect(second.storePendingTotal).toBe(1); + expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1); + } finally { + await store.quit(); + } + }); + + redisTest( + "mixed set: one pending and one already complete blocks the run once", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_pending"), status: "PENDING" }); + await store.createIfAbsent({ record: record("w_done"), status: "PENDING" }); + await store.complete({ waitpointId: "w_done", completion: completion() }); + + const result = await store.registerBlocks({ + runId: RUN_ID, + edges: [edge("w_pending"), edge("w_done")], + }); + + expect(result.pendingOfRequested).toBe(1); + expect(result.storePendingTotal).toBe(1); + expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_done"]); + } finally { + await store.quit(); + } + } + ); +}); + +describe("multi-index merge, end to end into the executor shape", () => { + redisTest( + "a run blocked on one waitpoint at two indexes resolves to two entries", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ + record: record("w_child", { type: "RUN", completedByTaskRunId: "run_child" }), + status: "PENDING", + }); + + await store.registerBlocks({ + runId: RUN_ID, + edges: [ + edge("w_child", { batchIndex: 0, batchId: "batch_1", type: "RUN" }), + edge("w_child", { batchIndex: 2, batchId: "batch_1", type: "RUN" }), + ], + }); + + const completed = await store.complete({ + waitpointId: "w_child", + completion: completion({ output: null }), + }); + // The cross-shard fact this test claims to prove: two registers for the same + // waitpoint at different indexes fanned out into two distinct watcher entries. + expect( + completed.watchers.map((w) => w.batchIndex).sort((a, b) => (a ?? 0) - (b ?? 0)) + ).toEqual([0, 2]); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_child", + completion: completed.completion!, + }); + + const state = await store.readBlockState(RUN_ID); + expect(state.pendingIds).toEqual([]); + expect(state.deliveredIds).toEqual(["w_child"]); + + // Derive the cycle's ordered id list the way the read path does: keep only edges + // that carry a batch index, sort ascending, map to id. Derived inline on purpose — + // another lane owns the order rule and its resolver, and this test's job is to + // prove the COORDINATOR preserved the edge multiplicity across two shards, not to + // own that rule. + const order = state.edges + .filter((e) => e.batchIndex !== undefined && e.batchIndex !== null) + .sort((a, b) => a.batchIndex! - b.batchIndex!) + .map((e) => e.waitpointId); + + // One waitpoint, two edges, so the id repeats — that repeat is what expands into + // two entries for the executor, and losing it would silently drop a batch item. + expect(order).toEqual(["w_child", "w_child"]); + expect(state.edges.map((e) => e.edgeId).sort()).toEqual(["w_child#0", "w_child#2"]); + expect(state.edges.every((e) => e.batchId === "batch_1")).toBe(true); + } finally { + await store.quit(); + } + } + ); +}); + +describe("the resume cycle drains and can start again", () => { + redisTest("a second wait on the same waitpoint blocks nothing", async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] }); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + await store.deliverCompletion({ + runId: RUN_ID, + waitpointId: "w_a", + completion: completed.completion!, + }); + + const first = await store.readBlockState(RUN_ID); + await store.clearBlockState({ runId: RUN_ID, edgeIds: first.edges.map((e) => e.edgeId) }); + + // Cycle two. The waitpoint is COMPLETED for good, so the register reports it and the + // run is never blocked. + const second = await store.registerBlocks({ + runId: RUN_ID, + edges: [edge("w_a", { batchIndex: 5 })], + }); + + expect(second.storePendingTotal).toBe(0); + expect(second.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]); + expect((await store.readBlockState(RUN_ID)).edges.map((e) => e.edgeId)).toEqual(["w_a#5"]); + } finally { + await store.quit(); + } + }); +}); + +// Every test above is a sequence of awaits. Redis guarantees atomicity WITHIN a script, so +// those tests can only ever prove single-script invariants. These races drive real +// concurrent calls (Promise.all over N copies) against the multi-script TypeScript +// sequences, and assert an invariant that holds regardless of who wins — never a timing. +describe("genuine concurrency", () => { + const CONCURRENCY = 8; + + redisTest( + "exactly one of N concurrent completers wins, and every caller sees its completion", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW }); + await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW }); + + const results = await Promise.all( + Array.from({ length: CONCURRENCY }, (_, i) => + store.complete({ + waitpointId: "w_a", + completion: completion({ output: { inline: `{"racer":${i}}` } }), + }) + ) + ); + + const winners = results.filter((r) => r.outcome === "completed"); + const losers = results.filter((r) => r.outcome === "already"); + expect(winners).toHaveLength(1); + expect(losers).toHaveLength(CONCURRENCY - 1); + + // Every caller, winner and losers alike, reads back the SAME stored completion. + const stored = winners[0]!.completion; + for (const r of results) { + expect(r.completion).toEqual(stored); + } + + // And every caller returns the full watcher list — a race must never truncate it. + for (const r of results) { + expect(r.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]); + } + } finally { + await store.quit(); + } + } + ); + + redisTest( + "a pre-existing registration survives N concurrent attempts to re-register its field", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + await store.createIfAbsent({ record: record("w_a"), status: "PENDING" }); + await store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: "span_first", + createdAt: NOW, + }); + + // Same run, same (absent) batch index as the registration above, so every one of + // these collides on the exact same watcher field. + await Promise.all( + Array.from({ length: CONCURRENCY }, (_, i) => + store.registerOrReport({ + waitpointId: "w_a", + runId: "run_1", + spanIdToComplete: `span_racer_${i}`, + createdAt: NOW, + }) + ) + ); + + const completed = await store.complete({ waitpointId: "w_a", completion: completion() }); + const forRun1 = completed.watchers.filter((w) => w.runId === "run_1"); + expect(forRun1).toHaveLength(1); + expect(forRun1[0]!.spanIdToComplete).toBe("span_first"); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "exactly one of N concurrent idempotency-keyed creators wins, and every loser cleans up", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + const probe = createRedisClient(redisOptions); + try { + const ids = Array.from({ length: CONCURRENCY }, () => generateWaitpointId("MANUAL")); + + const results = await Promise.all( + ids.map((id) => + store.createWithIdempotencyKey({ + record: record(id, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }), + environmentId: ENV_ID, + idempotencyKey: "key-1", + }) + ) + ); + + const winners = results.filter((r) => r.created); + expect(winners).toHaveLength(1); + + const winnerId = winners[0]!.waitpointId; + for (const r of results) { + expect(r.waitpointId).toBe(winnerId); + } + expect(await probe.exists(`wp:{${winnerId}}`)).toBe(1); + + for (const id of ids) { + if (id === winnerId) continue; + expect(await probe.exists(`wp:{${id}}`)).toBe(0); + expect(await probe.exists(`wp:{${id}}:w`)).toBe(0); + } + } finally { + probe.disconnect(); + await store.quit(); + } + } + ); + + redisTest( + "registerBlocks racing complete never leaves a waitpoint double-booked or the pending count negative", + async ({ redisOptions }) => { + const store = coordinator(redisOptions); + try { + for (let i = 0; i < 30; i++) { + const waitpointId = `w_race_${i}`; + const runId = `run_race_${i}`; + await store.createIfAbsent({ record: record(waitpointId), status: "PENDING" }); + + // Two edges for the SAME waitpoint: registerBlocks registers them one at a + // time, so a concurrent complete() has a real window to land between the two + // registrations — the exact straddle that makes absorbBlockers' per-group + // reported/unreported split matter, rather than racing a single all-or-nothing + // group. + const [blocked] = await Promise.all([ + store.registerBlocks({ + runId, + edges: [edge(waitpointId, { batchIndex: 0 }), edge(waitpointId, { batchIndex: 1 })], + }), + store.complete({ waitpointId, completion: completion() }), + ]); + + const state = await store.readBlockState(runId); + const delivered = state.deliveredIds.includes(waitpointId); + const pending = state.pendingIds.includes(waitpointId); + + expect(delivered && pending).toBe(false); + expect(blocked.storePendingTotal).toBeGreaterThanOrEqual(0); + expect(blocked.storePendingTotal).toBeLessThanOrEqual(1); + } + } finally { + await store.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts new file mode 100644 index 00000000000..723552c57ab --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts @@ -0,0 +1,538 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import { parseWaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { + assertSingleSlot, + edgeField, + idempotencyKey, + runBlockKeys, + waitpointKeys, + watcherField, +} from "./keys.js"; +import { registerWaitpointCommands } from "./scripts.js"; + +/** The values written into a record's `status` field. Uppercase, and never a token. */ +export type WaitpointStatus = "PENDING" | "COMPLETED"; + +/** Every script this coordinator may invoke. The wrapper below is the only entry point. */ +type ScriptName = + | "wpCreateIfAbsent" + | "wpRegisterOrReport" + | "wpComplete" + | "wpIdemReserve" + | "wpDiscard" + | "runAbsorbBlockers" + | "runDeliverCompletion" + | "runReadBlockState" + | "runClear"; + +/** + * The immutable half of a waitpoint, written once at creation. Carries every field the + * legacy-shaped return types need, including the two that gate the executor-visible + * idempotency key and the token surface. + */ +export type WaitpointRecordInput = { + id: string; + friendlyId: string; + type: "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + environmentId: string; + projectId: string; + createdAt: string; + updatedAt: string; + userProvidedIdempotencyKey: boolean; + tags: string[]; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: string; + completedAfter?: string; + completedByTaskRunId?: string; + completedByBatchId?: string; +}; + +/** + * A stored output: a small inline value, an already-offloaded reference, or null when the + * value is re-derivable from a business fact and is therefore never copied forward. + */ +export type WaitpointCompletionOutput = { inline: string } | { ref: string } | null; + +/** + * The completion half of a waitpoint, written at the flip. + * + * This is the coordinator's OWN type, deliberately not a projection of any frozen record + * type. The store treats a completion as an opaque blob: it writes it, returns it, and + * never inspects a field. Whoever owns the read-time resolver maps between this and the + * frozen record shape, so the two can evolve without a type dependency in either + * direction. + */ +export type WaitpointCompletion = { + /** ISO 8601. */ + completedAt: string; + outputType: string; + outputIsError: boolean; + output: WaitpointCompletionOutput; +}; + +export type WatcherEntry = { + runId: string; + batchIndex?: number; + spanIdToComplete?: string; + createdAt: string; +}; + +export type CreateIfAbsentResult = + | { outcome: "created" } + | { + outcome: "exists"; + record: WaitpointRecordInput; + status: WaitpointStatus; + completion?: WaitpointCompletion; + }; + +export type RegisterOrReportResult = + | { outcome: "registered" } + | { outcome: "completed"; completion?: WaitpointCompletion }; + +export type CompleteResult = { + outcome: "completed" | "already"; + completion?: WaitpointCompletion; + watchers: WatcherEntry[]; +}; + +/** + * One run-to-waitpoint edge. The metadata a frozen return type — an existing API response + * shape this store must keep reproducing — needs travels here. + */ +export type BlockEdge = { + waitpointId: string; + batchIndex?: number | null; + batchId?: string; + spanIdToComplete?: string; + createdAt: string; + type: WaitpointRecordInput["type"]; + completedAfter?: string; + // Set when the register step already reported this waitpoint COMPLETED. The box, not + // `completion`, carries the "reported" fact: box present + no completion means + // COMPLETED-with-no-envelope, box absent means never reported. + reported?: { completion?: WaitpointCompletion }; +}; + +export type AbsorbResult = { + /** + * How many DISTINCT requested ids were still pending. Equivalent to the count the + * previous path took over this call's ids, which was a COUNT over waitpoint rows — so + * two edges for one waitpoint contribute one. This is the number a caller should use to + * keep today's block-time gate unchanged. + */ + pendingOfRequested: number; + /** + * The run's whole pending set, counting STORE-RESIDENT blockers only. A run can also be + * blocked by a legacy waitpoint, which this number cannot see, so it is never on its own + * a decision to resume. + */ + storePendingTotal: number; + alreadyDelivered: Array<{ waitpointId: string; completion?: WaitpointCompletion }>; +}; + +// absorbBlockers strips `reported` before writing the edge blob, so a value read back +// here can never carry it — Omit says so instead of inheriting a field that is always +// undefined. +export type BlockStateEdge = Omit & { edgeId: string }; + +export type BlockState = { + pendingIds: string[]; + deliveredIds: string[]; + edges: BlockStateEdge[]; +}; + +export class WaitpointNotFoundError extends Error { + constructor(waitpointId: string) { + super(`Waitpoint ${waitpointId} is not present in the store`); + this.name = "WaitpointNotFoundError"; + } +} + +export type WaitpointStoreCoordinatorOptions = { + redisOptions: RedisOptions; + logger?: Logger; +}; + +// Lua returns '' for an absent value, never nil, because every reply slot is coerced to +// keep the array from truncating. So a nullish check would not fire and JSON.parse('') +// throws. One helper, used at every decode site. +function parseJson(raw: string | undefined): T | undefined { + return raw ? (JSON.parse(raw) as T) : undefined; +} + +export class WaitpointStoreCoordinator { + private readonly redis: Redis; + private readonly logger: Logger; + #quit?: Promise; + + constructor(options: WaitpointStoreCoordinatorOptions) { + this.logger = options.logger ?? new Logger("WaitpointStoreCoordinator", "debug"); + this.redis = createRedisClient(options.redisOptions, { + onError: (error) => + this.logger.error("WaitpointStoreCoordinator redis client error", { error }), + }); + registerWaitpointCommands(this.redis); + } + + // Idempotent and error-swallowing: every test calls this in a finally, and a double quit + // must never mask the real assertion failure. + async quit(): Promise { + if (!this.#quit) { + this.#quit = this.redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * The ONLY way this class invokes a script. Routing every call through one place is what + * makes the single-slot guard un-forgettable: a method added later cannot reach a script + * without passing its keys through this assertion. + * + * Every script's signature is (...keys, ...argv) => string[], so one cast covers them + * all. The typed RedisCommander augmentation in scripts.ts documents each shape. + */ + #call(script: ScriptName, keys: string[], ...argv: string[]): Promise { + assertSingleSlot(script, keys); + const command = this.redis[script] as (...args: string[]) => Promise; + return command.call(this.redis, ...keys, ...argv); + } + + /** + * Exposed for the guard's own test. Delegates through #call rather than calling + * assertSingleSlot directly, so a mutation to the guard inside #call fails this test too + * — not only the tests that happen to exercise a real script. + * + * With cross-tag (invalid) keys, assertSingleSlot throws synchronously inside #call, + * before any promise exists, and that throw propagates straight out of this method. With + * same-tag (valid) keys, #call would go on to dispatch a real script call; this method + * never returns or awaits that promise, and swallows whatever it eventually settles to, + * so a valid-key call here can never surface as an unhandled rejection in the caller. + */ + assertKeysForTest(operation: string, keys: string[]): void { + this.#call(operation as ScriptName, keys).catch(() => undefined); + } + + async createIfAbsent(args: { + record: WaitpointRecordInput; + status: WaitpointStatus; + completion?: WaitpointCompletion; + }): Promise { + const keys = waitpointKeys(args.record.id); + + const reply = await this.#call( + "wpCreateIfAbsent", + [keys.record], + JSON.stringify(args.record), + args.status, + args.completion ? JSON.stringify(args.completion) : "" + ); + + if (reply[0] === "created") { + return { outcome: "created" }; + } + + // reply[1] is '' only if the record hash exists with no 'r' field, which should never + // happen — but ?? never fires on '', so a bare JSON.parse('') would throw an + // undiagnosable SyntaxError instead of naming the waitpoint. + const record = parseJson(reply[1]); + if (!record) { + throw new Error(`Waitpoint ${args.record.id} exists in the store with no record blob`); + } + + return { + outcome: "exists", + record, + status: reply[2] === "COMPLETED" ? "COMPLETED" : "PENDING", + completion: parseJson(reply[3]), + }; + } + + async registerOrReport(args: { + waitpointId: string; + runId: string; + batchIndex?: number | null; + spanIdToComplete?: string; + createdAt: string; + }): Promise { + const keys = waitpointKeys(args.waitpointId); + + // batchIndex is nullable at the boundary (matching the column) and undefined inside, + // because JSON.stringify drops an undefined field but keeps a null one. + const watcher: WatcherEntry = { + runId: args.runId, + batchIndex: args.batchIndex ?? undefined, + spanIdToComplete: args.spanIdToComplete, + createdAt: args.createdAt, + }; + + const reply = await this.#call( + "wpRegisterOrReport", + [keys.record, keys.watchers], + watcherField(args.runId, args.batchIndex), + JSON.stringify(watcher) + ); + + if (reply[0] === "missing") { + throw new WaitpointNotFoundError(args.waitpointId); + } + if (reply[0] === "completed") { + return { outcome: "completed", completion: parseJson(reply[1]) }; + } + + return { outcome: "registered" }; + } + + async complete(args: { + waitpointId: string; + completion: WaitpointCompletion; + }): Promise { + const keys = waitpointKeys(args.waitpointId); + + const reply = await this.#call( + "wpComplete", + [keys.record, keys.watchers], + JSON.stringify(args.completion) + ); + + if (reply[0] === "missing") { + throw new WaitpointNotFoundError(args.waitpointId); + } + + return { + outcome: reply[0] as "completed" | "already", + completion: parseJson(reply[1]), + watchers: reply.slice(2).map((entry) => JSON.parse(entry) as WatcherEntry), + }; + } + + /** + * Create a waitpoint under an idempotency key. + * + * The reservation and the record sit under different hash tags, so no script spans + * them. That makes the ORDER load-bearing: create first, then reserve. + * + * Reserve-first would mean a crash between the two steps leaves a reservation naming a + * waitpoint that does not exist. Every later request with that key loses the + * reservation, blocks on the winner's id, and throws when it registers — correctly, but + * forever, because an idempotency key commonly carries no expiry to clear it. + * + * Create-first inverts the failure: a crash leaves an orphan record that nothing ever + * referenced, because its id is random and unpublished. No caller hangs, but nothing + * currently reclaims that record either: the backstop collector the wider plan + * describes is keyed off a run's status, and this orphan has no owning run, so that + * collector never sees it. The record is harmless — inert, unreferenced, never + * returned to anyone — but it is a real leak until a later ticket adds a reaper for + * standalone idempotency-keyed orphans specifically. + */ + async createWithIdempotencyKey(args: { + record: WaitpointRecordInput; + environmentId: string; + idempotencyKey: string; + // `created` means THIS CALL won the reservation, not that the id is new. A retry by the + // original creator reports false, because the reservation it is losing to is its own. A + // caller must not gate one-time side effects on it without handling that. + }): Promise<{ waitpointId: string; created: boolean }> { + // Standalone ids only. The discard below deletes this call's own record, and that is + // only safe because a freshly minted id was never handed out, so nothing can reference + // it. A RUN or BATCH id is DERIVED from its anchor, so any caller can recompute it and + // register a watcher on it — discarding one could delete a record already in use. + const parsed = parseWaitpointId(args.record.id); + if (parsed.format !== "b32hexW" || (parsed.type !== "DATETIME" && parsed.type !== "MANUAL")) { + throw new Error( + `createWithIdempotencyKey requires a freshly minted DATETIME or MANUAL id, got ${args.record.id}` + ); + } + + await this.createIfAbsent({ record: args.record, status: "PENDING" }); + + const expiresAtMs = args.record.idempotencyKeyExpiresAt + ? String(new Date(args.record.idempotencyKeyExpiresAt).getTime()) + : ""; + + const reply = await this.#call( + "wpIdemReserve", + [idempotencyKey(args.environmentId, args.idempotencyKey)], + args.record.id, + expiresAtMs + ); + + if (reply[0] === "reserved") { + return { waitpointId: args.record.id, created: true }; + } + + const winner = reply[1]; + if (winner !== args.record.id) { + // Safe to discard: this id is random and was never handed to any caller, so no + // watcher can reference it. Both keys share the record's tag. + const keys = waitpointKeys(args.record.id); + await this.#call("wpDiscard", [keys.record, keys.watchers]); + } + + return { waitpointId: winner, created: false }; + } + + async absorbBlockers(args: { runId: string; edges: BlockEdge[] }): Promise { + const keys = runBlockKeys(args.runId); + + // No fast path for an empty list: storePendingTotal is defined as the run's WHOLE + // store-resident pending set, so it has to be read even when nothing is requested. + const argv: string[] = [String(args.edges.length)]; + for (const item of args.edges) { + const { reported, ...stored } = item; + const reportedFlag = reported !== undefined ? "1" : "0"; + const reportedJson = reported?.completion ? JSON.stringify(reported.completion) : ""; + argv.push( + item.waitpointId, + edgeField(item.waitpointId, item.batchIndex), + JSON.stringify(stored), + reportedFlag, + reportedJson + ); + } + + const reply = await this.#call("runAbsorbBlockers", [keys.pend, keys.done, keys.edge], ...argv); + + const alreadyDelivered: AbsorbResult["alreadyDelivered"] = []; + for (let i = 2; i < reply.length; i += 2) { + alreadyDelivered.push({ + waitpointId: reply[i]!, + completion: parseJson(reply[i + 1]), + }); + } + + return { + pendingOfRequested: Number(reply[0]), + storePendingTotal: Number(reply[1]), + alreadyDelivered, + }; + } + + /** + * Block a run on a set of waitpoints. + * + * Register on every waitpoint's own shard FIRST, then absorb on the run's shard. The + * order is the protocol: a completion that lands in between finds the watcher already + * registered, so it delivers onto the run's shard, and the absorb sees that delivery and + * never marks the waitpoint pending. + * + * The register keys the decision to skip the pending set on OUTCOME, never on whether a + * completion envelope came back — a waitpoint can be reported COMPLETED with none. + * + * A throw partway through (a missing waitpoint) intentionally leaves any + * already-registered watchers in place rather than unwinding them. That's safe: a later + * `complete` on one of those waitpoints still delivers correctly, and if it lands before + * this run ever retries `registerBlocks`, the stray `done` entry it writes is inert until + * a future absorb or `clearBlockState`'s reconcile reads it — never a false resume. + */ + async registerBlocks(args: { runId: string; edges: BlockEdge[] }): Promise { + const registered: BlockEdge[] = []; + + for (const item of args.edges) { + const result = await this.registerOrReport({ + waitpointId: item.waitpointId, + runId: args.runId, + batchIndex: item.batchIndex, + spanIdToComplete: item.spanIdToComplete, + createdAt: item.createdAt, + }); + + registered.push( + result.outcome === "completed" + ? { ...item, reported: { completion: result.completion } } + : item + ); + } + + return this.absorbBlockers({ runId: args.runId, edges: registered }); + } + + async deliverCompletion(args: { + runId: string; + waitpointId: string; + completion: WaitpointCompletion; + }): Promise<{ storePendingTotal: number }> { + const keys = runBlockKeys(args.runId); + + const reply = await this.#call( + "runDeliverCompletion", + [keys.pend, keys.done], + args.waitpointId, + JSON.stringify(args.completion) + ); + + return { storePendingTotal: Number(reply[0]) }; + } + + async readBlockState(runId: string): Promise { + const keys = runBlockKeys(runId); + const reply = await this.#call("runReadBlockState", [keys.pend, keys.done, keys.edge]); + + // Slots 0 and 1 are true element counts, but slot 2 is the FLAT length of the edge + // HGETALL — two entries per edge, field then value. The cursor arithmetic below relies + // on that asymmetry, so do not "normalise" it without changing the Lua too. + const pendCount = Number(reply[0]); + const doneCount = Number(reply[1]); + const edgeCount = Number(reply[2]); + + let cursor = 3; + const pendingIds = reply.slice(cursor, cursor + pendCount); + cursor += pendCount; + const deliveredIds = reply.slice(cursor, cursor + doneCount); + cursor += doneCount; + + const edges: BlockStateEdge[] = []; + for (let i = 0; i < edgeCount; i += 2) { + const edgeId = reply[cursor + i]!; + // An edge value is always a non-empty JSON.stringify, so a missing slot here means + // the cursor walked off the end of the reply. That must fail loudly, not decode a + // BlockEdge with no waitpointId — the exact off-by-one this task's arithmetic guards + // against. + const edgeJson = reply[cursor + i + 1]; + if (!edgeJson) { + throw new Error( + `readBlockState(${runId}): missing edge payload at reply index ${cursor + i + 1}` + ); + } + const stored = JSON.parse(edgeJson) as BlockEdge; + edges.push({ ...stored, edgeId }); + } + + return { pendingIds, deliveredIds, edges }; + } + + /** + * Drain one cycle's edges, or clear the run entirely when no edge ids are given. + * + * The selective form RECONCILES: any pending or delivered entry that no surviving edge + * references goes too, not only the named ones. See runClear in scripts.ts for why. + */ + async clearBlockState(args: { + runId: string; + edgeIds?: string[]; + }): Promise<{ outcome: "cleared" | "drained" | "noop" }> { + // `omitted` and `explicitly empty` must not collapse onto each other: the Lua's + // n === 0 means "clear the whole run", so an omitted edgeIds stays the terminal clear, + // but a caller that computed zero edges to drain gets a genuine no-op that never + // reaches Redis. + if (args.edgeIds && args.edgeIds.length === 0) { + return { outcome: "noop" }; + } + + const keys = runBlockKeys(args.runId); + const edgeIds = args.edgeIds ?? []; + + const reply = await this.#call( + "runClear", + [keys.pend, keys.done, keys.edge], + String(edgeIds.length), + ...edgeIds + ); + + return { outcome: reply[0] as "cleared" | "drained" }; + } +} diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts index 2c54e4c20c0..2c98edf6866 100644 --- a/internal-packages/run-engine/src/index.ts +++ b/internal-packages/run-engine/src/index.ts @@ -38,3 +38,26 @@ export type { ProcessBatchItemCallback, BatchCompletionCallback, } from "./batch-queue/types.js"; + +// Waitpoint store coordinator. Exported but not yet wired: a later ticket routes +// WaitpointSystem onto it behind a per-organisation flag. +export { + WaitpointStoreCoordinator, + WaitpointNotFoundError, +} from "./engine/waitpointCoordinator/storeCoordinator.js"; +export type { + AbsorbResult, + BlockEdge, + BlockState, + BlockStateEdge, + CompleteResult, + CreateIfAbsentResult, + RegisterOrReportResult, + WaitpointCompletion, + WaitpointCompletionOutput, + WaitpointRecordInput, + WaitpointStatus, + WaitpointStoreCoordinatorOptions, + WatcherEntry, +} from "./engine/waitpointCoordinator/storeCoordinator.js"; +export { WaitpointKeyTagError } from "./engine/waitpointCoordinator/keys.js"; diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts index 2e3ba4d83a5..b5ea7a51971 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.test.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts @@ -11,13 +11,20 @@ import { RUN_OPS_ID_VERSION, RUN_OPS_ID_VERSION_2, RUN_OPS_ID_VERSION_INDEX, + WAITPOINT_ID_TYPE_INDEX, + WAITPOINT_ID_VERSION, base32hexDecode, base32hexEncode, + deriveWaitpointIdFromAnchor, + generateFriendlyId, generateRunOpsId, generateRunOpsIdV2, + generateWaitpointId, parseRunId, parseRunOpsIdBody, parseRunOpsIdV2Body, + parseWaitpointId, + type WaitpointIdType, } from "./friendlyId.js"; /** Every legal gen-2 shard char: the full DNS-safe lowercase range. */ @@ -410,3 +417,158 @@ describe("parseRunId — v2 arm", () => { expect(parseRunId(`waitpoint_${generateRunOpsIdV2("a")}`).format).toBe("legacy"); }); }); + +describe("waitpoint ids: run-ops format with version char w", () => { + it("mints a 26-char body per type, with the type char at index 24 and version w at 25", () => { + const cases: Array<[WaitpointIdType, string]> = [ + ["RUN", "r"], + ["BATCH", "b"], + ["DATETIME", "d"], + ["MANUAL", "m"], + ]; + + for (const [type, typeChar] of cases) { + const body = generateWaitpointId(type); + expect(body.length).toBe(RUN_OPS_ID_LENGTH); + expect(body[WAITPOINT_ID_TYPE_INDEX]).toBe(typeChar); + expect(body[RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION); + } + }); + + it("round-trips every type char through parseWaitpointId", () => { + for (const type of ["RUN", "BATCH", "DATETIME", "MANUAL"] as WaitpointIdType[]) { + const parsed = parseWaitpointId(generateWaitpointId(type)); + expect(parsed.format).toBe("b32hexW"); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.type).toBe(type); + } + }); + + it("classifies both the prefixed and the bare form identically", () => { + const body = generateWaitpointId("MANUAL"); + const bare = parseWaitpointId(body); + const prefixed = parseWaitpointId(`waitpoint_${body}`); + expect(bare).toEqual(prefixed); + expect(bare).toEqual({ format: "b32hexW", type: "MANUAL", timestamp: expect.any(Date) }); + }); + + it("recovers the mint timestamp from the core", () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-08-21T12:00:00.000Z")); + const parsed = parseWaitpointId(generateWaitpointId("DATETIME")); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.timestamp.toISOString()).toBe("2026-08-21T12:00:00.000Z"); + } finally { + vi.useRealTimers(); + } + }); + + it("classifies every legacy shape as legacy", () => { + const legacy = [ + WaitpointId.generate().id, + WaitpointId.generate().friendlyId, + generateFriendlyId("waitpoint"), + "", + "waitpoint_", + "a".repeat(27), + "a".repeat(26), + ]; + + for (const id of legacy) { + expect(parseWaitpointId(id).format).toBe("legacy"); + } + }); + + it("rejects a 26-char body whose version is w but whose type char is not r/b/d/m", () => { + const body = generateWaitpointId("RUN"); + const bad = `${body.slice(0, WAITPOINT_ID_TYPE_INDEX)}x${WAITPOINT_ID_VERSION}`; + expect(parseWaitpointId(bad).format).toBe("legacy"); + }); + + it("rejects a body whose core is outside the base32hex alphabet", () => { + const body = generateWaitpointId("RUN"); + // "w" is outside [0-9a-v], so the core no longer decodes. + expect(parseWaitpointId(`w${body.slice(1)}`).format).toBe("legacy"); + }); + + it("never parses a run id as a waitpoint id, or the reverse", () => { + expect(parseWaitpointId(generateRunOpsId()).format).toBe("legacy"); + expect(parseWaitpointId(generateRunOpsIdV2("7")).format).toBe("legacy"); + expect(parseRunId(`run_${generateWaitpointId("RUN")}`).format).toBe("legacy"); + }); + + it("rejects a well-formed waitpoint body wearing a foreign prefix", () => { + const body = `${"0".repeat(24)}rw`; // valid core + RUN type char + version w + expect(parseWaitpointId(`run_${body}`).format).toBe("legacy"); + expect(parseWaitpointId(`batch_${body}`).format).toBe("legacy"); + expect(parseWaitpointId(`waitpoint_${body}`)).toEqual({ + format: "b32hexW", + type: "RUN", + timestamp: expect.any(Date), + }); + expect(parseWaitpointId(body).format).toBe("b32hexW"); + }); + + it("handles a bare body that happens to contain an underscore sanely (never throws, never misclassifies)", () => { + const body = generateWaitpointId("BATCH"); + const withUnderscore = `_${body.slice(1)}`; + expect(() => parseWaitpointId(withUnderscore)).not.toThrow(); + // "_" is outside the base32hex alphabet, so this can never be a real waitpoint id. + expect(parseWaitpointId(withUnderscore).format).toBe("legacy"); + }); +}); + +describe("deriveWaitpointIdFromAnchor", () => { + it("is deterministic: the same anchor and type always give the same id", () => { + const anchor = `run_${generateRunOpsId("us-east-1")}`; + const first = deriveWaitpointIdFromAnchor(anchor, "RUN"); + expect(first).toBeDefined(); + expect(first).toBe(deriveWaitpointIdFromAnchor(anchor, "RUN")); + }); + + it("shares the anchor's 24-char core and replaces the region and version chars", () => { + const anchorBody = generateRunOpsId("us-east-1"); + const derived = deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN"); + expect(derived).toBeDefined(); + expect(derived!.slice(0, WAITPOINT_ID_TYPE_INDEX)).toBe( + anchorBody.slice(0, WAITPOINT_ID_TYPE_INDEX) + ); + expect(derived![WAITPOINT_ID_TYPE_INDEX]).toBe("r"); + expect(derived![RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION); + }); + + it("accepts a bare anchor body as well as a prefixed one", () => { + const anchorBody = generateRunOpsId(); + expect(deriveWaitpointIdFromAnchor(anchorBody, "RUN")).toBe( + deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN") + ); + }); + + it("accepts a gen-2 anchor", () => { + const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsIdV2("7")}`, "RUN"); + expect(derived).toBeDefined(); + expect(parseWaitpointId(derived!).format).toBe("b32hexW"); + }); + + it("derives a BATCH id from a run-ops format batch anchor", () => { + const derived = deriveWaitpointIdFromAnchor(`batch_${generateRunOpsId()}`, "BATCH"); + expect(derived).toBeDefined(); + const parsed = parseWaitpointId(derived!); + if (parsed.format !== "b32hexW") throw new Error("unreachable"); + expect(parsed.type).toBe("BATCH"); + }); + + it("returns undefined for a legacy anchor, so the caller falls back to a legacy mint", () => { + expect(deriveWaitpointIdFromAnchor(RunId.generate().friendlyId, "RUN")).toBeUndefined(); + expect(deriveWaitpointIdFromAnchor("run_", "RUN")).toBeUndefined(); + expect(deriveWaitpointIdFromAnchor("", "RUN")).toBeUndefined(); + }); + + it("gives a different id per type from one anchor", () => { + const anchor = `run_${generateRunOpsId()}`; + expect(deriveWaitpointIdFromAnchor(anchor, "RUN")).not.toBe( + deriveWaitpointIdFromAnchor(anchor, "BATCH") + ); + }); +}); diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index c468de65319..2f436b93a3e 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -238,6 +238,105 @@ export function parseRunId(id: string): ParsedRunId { return LEGACY_RUN_ID; } +// Waitpoint ids reuse the run-ops body layout — 24-char base32hex core, then a +// positional char, then a version char — so the body parses positionally instead of +// splitting on "_". Index 24 carries the TYPE (the slot a run uses for its region or +// shard char), which leaves room to move to a shard char under a later version. +export const WAITPOINT_ID_VERSION = "w"; +export const WAITPOINT_ID_TYPE_INDEX = RUN_OPS_ID_REGION_INDEX; + +export type WaitpointIdType = "RUN" | "BATCH" | "DATETIME" | "MANUAL"; + +// "w" sits OUTSIDE the base32hex alphabet [0-9a-v], so the version char can never be +// mistaken for a core char, and it can never collide with a numeric run generation. +const WAITPOINT_TYPE_CHARS: Readonly> = { + RUN: "r", + BATCH: "b", + DATETIME: "d", + MANUAL: "m", +}; + +const WAITPOINT_TYPES_BY_CHAR: Readonly> = { + r: "RUN", + b: "BATCH", + d: "DATETIME", + m: "MANUAL", +}; + +export type ParsedWaitpointId = + | { format: "b32hexW"; type: WaitpointIdType; timestamp: Date } + | { format: "legacy" }; + +const LEGACY_WAITPOINT_ID: ParsedWaitpointId = { format: "legacy" }; + +/** + * Mint a standalone waitpoint id body (26 chars, no prefix) for DATETIME and MANUAL: a + * fresh core, the type char, then the waitpoint version char. + */ +export function generateWaitpointId(type: WaitpointIdType): string { + return `${mintRunOpsIdCore()}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`; +} + +/** + * Derive the 1:1 waitpoint id body for a RUN or BATCH anchor by reusing the anchor's + * 24-char core. Pure, so create-if-absent is idempotent without a lock. Returns + * undefined when the anchor is not a run-ops id, which is the caller's signal to mint a + * legacy waitpoint instead. + * + * Only the core survives: the anchor's region or shard char and its version char are + * both replaced. So the anchor id is NOT recoverable from the waitpoint id — the reverse + * direction uses the completedBy* back-pointer. + */ +export function deriveWaitpointIdFromAnchor( + anchorId: string, + type: WaitpointIdType +): string | undefined { + const body = stripAnchorPrefix(anchorId); + if (!parseRunOpsIdBody(body) && !parseRunOpsIdV2Body(body)) { + return undefined; + } + + return `${body.slice(0, RUN_OPS_ID_CORE_LENGTH)}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`; +} + +/** + * Classify a waitpoint id. Accepts the prefixed (`waitpoint_`) and bare forms, but + * NOT another entity's prefix (`run_`, `batch_`, ...) — this is the discriminator a + * later ticket uses to route a possibly customer-supplied id, so a foreign prefix must + * classify legacy rather than have its body reinterpreted as a waitpoint id. Total: + * never throws. + */ +export function parseWaitpointId(id: string): ParsedWaitpointId { + const body = stripWaitpointIdPrefix(id); + if (body.length !== RUN_OPS_ID_LENGTH) return LEGACY_WAITPOINT_ID; + if (body[RUN_OPS_ID_VERSION_INDEX] !== WAITPOINT_ID_VERSION) return LEGACY_WAITPOINT_ID; + + const type = WAITPOINT_TYPES_BY_CHAR[body[WAITPOINT_ID_TYPE_INDEX] ?? ""]; + if (!type) return LEGACY_WAITPOINT_ID; + + const timestamp = parseRunOpsIdCoreTimestamp(body); + if (timestamp === undefined) return LEGACY_WAITPOINT_ID; + + return { format: "b32hexW", type, timestamp }; +} + +// Strip any `_` if present. Prefix-agnostic is correct ONLY here: the caller +// already knows anchorId names a run or batch anchor, so there is no foreign prefix to +// guard against. Do not reuse for parseWaitpointId — see stripWaitpointIdPrefix. +function stripAnchorPrefix(id: string): string { + const underscore = id.indexOf("_"); + return underscore === -1 ? id : id.slice(underscore + 1); +} + +const WAITPOINT_ID_PREFIX = "waitpoint_"; + +// Strip the `waitpoint_` prefix if present; any other prefix, or a bare body, is left +// as-is. Unlike stripAnchorPrefix, this must never strip a foreign prefix down to a body +// that then happens to pass the run-ops shape check. +function stripWaitpointIdPrefix(id: string): string { + return id.startsWith(WAITPOINT_ID_PREFIX) ? id.slice(WAITPOINT_ID_PREFIX.length) : id; +} + export function generateInternalId(): string { return cuid(); } From f866210388d9578b8a03cc218a1049ec9592d79a Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 09:50:49 +0200 Subject: [PATCH 07/28] feat(cli): experimental --local-bundle deploy mode (#4331) Adds an experimental `--local-bundle` flag to native build deployments: the project is installed and bundled on the local machine (exactly like in the depot path) and only the resulting build context is uploaded. The remote build then runs just the container image build. ### Design - The uploaded artifact is the same build context classic deploys produce: bundled output, a synthesized package.json with the resolved externals, build.json, and the generated Containerfile. The bundle is secret-free: build.json is deliberately scrubbed because it is copied into the image, and build-arg values never enter the bundle at all. - Build-arg values are sent with the deployment initialization request instead, stored encrypted (aes-256-gcm) in a new `WorkerDeployment.buildEnvVars` column, and cleared on every terminal status transition. They exist at rest only for the active build window, always encrypted. - A dedicated `GET /api/v1/deployments/:id/build-env-vars` endpoint returns the decrypted values to the same principals that can already read the environment's variables. It answers with an empty record for deployments without stored values or in a terminal state, keeping secret access to a single auditable route. - Size limits are enforced server side and pre-checked client side. If the server does not acknowledge storing the values, the CLI fails fast instead of letting the remote build run without them. - A `--from-bundle ` mode builds a deployment image straight from such a bundle directory, skipping config loading and bundling entirely. In attach mode it fetches the stored build-arg values through the new endpoint. - Env var syncing (the `syncEnvVars` extension) happens client side, before the deployment initializes, since the remote side never sees the unscrubbed manifest. - Bundle artifacts use a distinct type and storage prefix so the server can always distinguish them from source uploads. --- .changeset/local-bundle-deploy.md | 6 + apps/webapp/app/env.server.ts | 13 + apps/webapp/app/routes/api.v1.artifacts.ts | 3 + ...eployments.$deploymentId.build-env-vars.ts | 98 ++ .../webapp/app/services/platform.v3.server.ts | 5 + .../app/v3/services/artifacts.server.ts | 7 +- ...eateDeploymentBackgroundWorkerV4.server.ts | 10 +- .../app/v3/services/deployment.server.ts | 4 +- .../app/v3/services/failDeployment.server.ts | 3 +- .../v3/services/finalizeDeployment.server.ts | 2 + .../services/initializeDeployment.server.ts | 36 + .../v3/services/timeoutDeployment.server.ts | 2 + apps/webapp/vite.config.ts | 2 + .../migration.sql | 2 + .../database/prisma/schema.prisma | 3 + packages/cli-v3/src/apiClient.ts | 15 + packages/cli-v3/src/commands/deploy.ts | 1037 ++++++++++++++++- .../cli-v3/src/deploy/bundleArchive.test.ts | 106 ++ packages/cli-v3/src/deploy/bundleArchive.ts | 40 + packages/core/src/v3/schemas/api.ts | 30 +- 20 files changed, 1413 insertions(+), 11 deletions(-) create mode 100644 .changeset/local-bundle-deploy.md create mode 100644 apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts create mode 100644 internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql create mode 100644 packages/cli-v3/src/deploy/bundleArchive.test.ts create mode 100644 packages/cli-v3/src/deploy/bundleArchive.ts diff --git a/.changeset/local-bundle-deploy.md b/.changeset/local-bundle-deploy.md new file mode 100644 index 00000000000..cb9cfdfee17 --- /dev/null +++ b/.changeset/local-bundle-deploy.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Add an experimental `--local-bundle` deploy flag that runs the install and bundling steps on your machine and uploads only the build output; the image is still built remotely. Useful when your project's install step needs tooling or credentials that only exist locally. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c9179306124..496c6e3d9a6 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -816,6 +816,19 @@ const EnvironmentSchema = z .number() .int() .default(60 * 1000 * 15), // 15 minutes + DEPLOYMENT_CONTEXT_ARTIFACT_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(100 * 1024 * 1024), // 100MB + DEPLOYMENT_BUNDLE_ARTIFACT_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(100 * 1024 * 1024), // 100MB + DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES: z.coerce + .number() + .int() + .default(128 * 1024), // 128KB + DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS: z.coerce.number().int().default(400), // When enabled, reject deploys made by v3 CLI versions (i.e. payloads that // omit the `type` field). v4 CLI versions always send `type` ("MANAGED" or "V1"), diff --git a/apps/webapp/app/routes/api.v1.artifacts.ts b/apps/webapp/app/routes/api.v1.artifacts.ts index a706f9e04ef..12c2a10a9ea 100644 --- a/apps/webapp/app/routes/api.v1.artifacts.ts +++ b/apps/webapp/app/routes/api.v1.artifacts.ts @@ -64,6 +64,9 @@ export async function action({ request }: ActionFunctionArgs) { case "deployment_context": errorMessage = `Artifact size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB. Make sure you are in the correct directory of your Trigger.dev project. Reach out to us if you are seeing this error consistently.`; break; + case "deployment_bundle": + errorMessage = `Bundle size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB. Reach out to us if you are seeing this error consistently.`; + break; default: body.data.type satisfies never; errorMessage = `Artifact size (${sizeMB} MB) exceeds the allowed limit of ${limitMB} MB`; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts new file mode 100644 index 00000000000..739046df13b --- /dev/null +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.build-env-vars.ts @@ -0,0 +1,98 @@ +import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { type GetDeploymentBuildEnvVarsResponseBody } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { decryptSecret, EncryptedSecretValueSchema } from "~/services/secrets/secretStore.server"; +import { FINAL_DEPLOYMENT_STATUSES } from "~/v3/services/failDeployment.server"; + +const ParamsSchema = z.object({ + deploymentId: z.string(), +}); + +// Secret material, deliberately separate from the main GET deployment endpoint. +export async function loader({ request, params }: LoaderFunctionArgs) { + const parsedParams = ParamsSchema.safeParse(params); + + if (!parsedParams.success) { + return json({ error: "Invalid params" }, { status: 400 }); + } + + try { + const authResult = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "deployments" }, + }); + + if (!authResult.ok) { + logger.info("Invalid or missing api key", { url: request.url }); + return json({ error: authResult.error }, { status: authResult.status }); + } + + const authenticatedEnv = authResult.authentication.environment; + + const { deploymentId } = parsedParams.data; + + const deployment = await prisma.workerDeployment.findFirst({ + where: { + friendlyId: deploymentId, + environmentId: authenticatedEnv.id, + }, + select: { + id: true, + status: true, + buildEnvVars: true, + }, + }); + + if (!deployment) { + return json({ error: "Deployment not found" }, { status: 404 }); + } + + logger.info("Build env vars read", { + deploymentId, + environmentId: authenticatedEnv.id, + projectId: authenticatedEnv.projectId, + status: deployment.status, + hasVars: deployment.buildEnvVars !== null, + }); + + // Never serve secrets for a build that is no longer active, even if a clear is still in flight + if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { + return json({ variables: {} } satisfies GetDeploymentBuildEnvVarsResponseBody, { + status: 200, + }); + } + + if (!deployment.buildEnvVars) { + return json({ variables: {} } satisfies GetDeploymentBuildEnvVarsResponseBody, { + status: 200, + }); + } + + // Present-but-unreadable must fail loud: an empty record would let the build run without its secrets + const envelope = EncryptedSecretValueSchema.safeParse(deployment.buildEnvVars); + + if (!envelope.success) { + logger.error("Stored build env vars are not a valid encrypted envelope", { + deploymentId, + environmentId: authenticatedEnv.id, + }); + return json( + { error: "The stored build environment variables could not be read. Retry the deploy." }, + { status: 500 } + ); + } + + const decrypted = await decryptSecret(env.ENCRYPTION_KEY, envelope.data); + const variables = z.record(z.string()).parse(JSON.parse(decrypted)); + + return json({ variables } satisfies GetDeploymentBuildEnvVarsResponseBody, { status: 200 }); + } catch (error) { + if (error instanceof Response) throw error; + logger.error("Failed to load deployment build env vars", { error }); + return json({ error: "Internal Server Error" }, { status: 500 }); + } +} diff --git a/apps/webapp/app/services/platform.v3.server.ts b/apps/webapp/app/services/platform.v3.server.ts index 01b1c7d4972..2b7e8fb3d4f 100644 --- a/apps/webapp/app/services/platform.v3.server.ts +++ b/apps/webapp/app/services/platform.v3.server.ts @@ -1095,6 +1095,7 @@ export async function enqueueBuild( options: { skipPromotion?: boolean; configFilePath?: string; + fromBundle?: boolean; } ) { if (!client) return undefined; @@ -1235,6 +1236,10 @@ export function isCloud(): boolean { return true; } + if (env.LOGIN_ORIGIN?.endsWith(".triggerlabs.dev")) { + return true; + } + if (process.env.CLOUD_ENV === "development" && process.env.NODE_ENV === "development") { return true; } diff --git a/apps/webapp/app/v3/services/artifacts.server.ts b/apps/webapp/app/v3/services/artifacts.server.ts index 9e82af51234..2d1ef190978 100644 --- a/apps/webapp/app/v3/services/artifacts.server.ts +++ b/apps/webapp/app/v3/services/artifacts.server.ts @@ -24,16 +24,19 @@ const objectStoreClient = const artifactKeyPrefixByType = { deployment_context: "deployments", + // The key prefix is the one bundle signal that survives schema skew + deployment_bundle: "bundles", } as const; const artifactBytesSizeLimitByType = { - deployment_context: 100 * 1024 * 1024, // 100MB + deployment_context: env.DEPLOYMENT_CONTEXT_ARTIFACT_SIZE_LIMIT_BYTES, + deployment_bundle: env.DEPLOYMENT_BUNDLE_ARTIFACT_SIZE_LIMIT_BYTES, } as const; export class ArtifactsService extends BaseService { private readonly bucket = env.ARTIFACTS_OBJECT_STORE_BUCKET; public createArtifact( - type: "deployment_context", + type: "deployment_context" | "deployment_bundle", authenticatedEnv: AuthenticatedEnvironment, contentLength?: number ) { diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index 305ee45ce25..d09707a0e83 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -1,9 +1,10 @@ import type { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; import { logger, tryCatch } from "@trigger.dev/core/v3"; -import type { - BackgroundWorker, - PrismaClientOrTransaction, - WorkerDeployment, +import { + Prisma, + type BackgroundWorker, + type PrismaClientOrTransaction, + type WorkerDeployment, } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { type TaskMetadataCache } from "~/services/taskMetadataCache.server"; @@ -313,6 +314,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { name: error.name, message: error.message, }, + buildEnvVars: Prisma.DbNull, }, }); diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index c67d7778568..7a891ae4f61 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -1,7 +1,7 @@ import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { BaseService } from "./baseService.server"; import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow"; -import { type WorkerDeployment, type Project, boundedIn } from "@trigger.dev/database"; +import { Prisma, type WorkerDeployment, type Project, boundedIn } from "@trigger.dev/database"; import { BuildServerMetadata, logger, @@ -227,6 +227,7 @@ export class DeploymentService extends BaseService { status: "CANCELED", canceledAt: new Date(), canceledReason: data?.canceledReason, + buildEnvVars: Prisma.DbNull, }, }), (error) => ({ @@ -339,6 +340,7 @@ export class DeploymentService extends BaseService { options: { skipPromotion?: boolean; configFilePath?: string; + fromBundle?: boolean; } ) { return fromPromise( diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 87b7618d76d..cb5c622b7b2 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,7 +1,7 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; @@ -49,6 +49,7 @@ export class FailDeploymentService extends BaseService { status: "FAILED", failedAt: new Date(), errorData: params.error, + buildEnvVars: Prisma.DbNull, }, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 0595cee1e2b..51f5b1e37c4 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -1,4 +1,5 @@ import type { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; +import { Prisma } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { updateEnvConcurrencyLimits } from "../runQueue.server"; @@ -76,6 +77,7 @@ export class FinalizeDeploymentService extends BaseService { deployedAt: new Date(), // Only add the digest, if any imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + buildEnvVars: Prisma.DbNull, }, }); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index c5b01c6084b..ee55d8bd8d6 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -6,6 +6,7 @@ import { import { customAlphabet } from "nanoid"; import { env } from "~/env.server"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { encryptSecret } from "~/services/secrets/secretStore.server"; import { logger } from "~/services/logger.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server"; @@ -268,6 +269,38 @@ export class InitializeDeploymentService extends BaseService { } : undefined; + let encryptedBuildEnvVars: Awaited> | undefined; + + if ( + payload.isNativeBuild && + payload.fromBundle && + payload.buildEnvVars && + Object.keys(payload.buildEnvVars).length > 0 + ) { + const buildEnvVars = payload.buildEnvVars; + + const keyCount = Object.keys(buildEnvVars).length; + if (keyCount > env.DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS) { + throw new ServiceValidationError( + `Build environment variable count (${keyCount}) exceeds the allowed limit of ${env.DEPLOYMENT_BUILD_ENV_VARS_MAX_KEYS}. Reach out to us if you are seeing this error consistently.` + ); + } + + const serialized = JSON.stringify(buildEnvVars); + const serializedBytes = Buffer.byteLength(serialized, "utf8"); + if (serializedBytes > env.DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES) { + const sizeKB = parseFloat((serializedBytes / 1024).toFixed(1)); + const limitKB = parseFloat( + (env.DEPLOYMENT_BUILD_ENV_VARS_SIZE_LIMIT_BYTES / 1024).toFixed(1) + ); + throw new ServiceValidationError( + `Build environment variables size (${sizeKB} KB) exceeds the allowed limit of ${limitKB} KB. Reach out to us if you are seeing this error consistently.` + ); + } + + encryptedBuildEnvVars = await encryptSecret(env.ENCRYPTION_KEY, serialized); + } + const buildServerMetadata: BuildServerMetadata | undefined = payload.isNativeBuild || payload.buildId ? { @@ -279,6 +312,7 @@ export class InitializeDeploymentService extends BaseService { skipPromotion: payload.skipPromotion, configFilePath: payload.configFilePath, skipEnqueue: payload.skipEnqueue, + fromBundle: payload.fromBundle, } : {}), } @@ -343,6 +377,7 @@ export class InitializeDeploymentService extends BaseService { projectId: environment.projectId, externalBuildData, buildServerMetadata, + buildEnvVars: encryptedBuildEnvVars, triggeredById: triggeredBy?.id, type: payload.type, imageReference: imageRef, @@ -373,6 +408,7 @@ export class InitializeDeploymentService extends BaseService { .enqueueBuild(environment, deployment, payload.artifactKey, { skipPromotion: payload.skipPromotion, configFilePath: payload.configFilePath, + fromBundle: payload.fromBundle, }) .orElse((error) => { logger.error("Failed to enqueue build", { diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index fa3de698e36..5e417a7863b 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -1,3 +1,4 @@ +import { Prisma } from "@trigger.dev/database"; import { logger } from "~/services/logger.server"; import { BaseService } from "./baseService.server"; import { commonWorker } from "../commonWorker.server"; @@ -45,6 +46,7 @@ export class TimeoutDeploymentService extends BaseService { status: "TIMED_OUT", failedAt: new Date(), errorData: { message: errorMessage, name: "TimeoutError" }, + buildEnvVars: Prisma.DbNull, }, }); diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index 56ddae17c02..967fd8fded3 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -75,6 +75,8 @@ export default defineConfig({ clientFiles: ["./app/entry.client.tsx", "./app/root.tsx", "./app/components/**/*.tsx"], ssrFiles: ["./app/entry.server.tsx", "./app/root.tsx"], }, + // In-build calls from local docker (e.g. the indexer) reach the dev webapp via this host + allowedHosts: ["host.docker.internal"], }, build: { sourcemap: true, diff --git a/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql b/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql new file mode 100644 index 00000000000..49e62e6e20d --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260722155558_add_worker_deployment_build_env_vars/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "buildEnvVars" JSONB; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 470f2c251c4..a77890930b1 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2271,6 +2271,9 @@ model WorkerDeployment { externalBuildData Json? buildServerMetadata Json? + /// Encrypted build-time env vars for pre-bundled (fromBundle) deploys, as an + /// EncryptedSecretValue envelope. Cleared when the deployment reaches a terminal status. + buildEnvVars Json? status WorkerDeploymentStatus @default(PENDING) type WorkerDeploymentType @default(V1) diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index fba2e52e1ee..8b9fd56eb1c 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -23,6 +23,7 @@ import { DevDisconnectResponseBody, EnvironmentVariableResponseBody, FailDeploymentResponseBody, + GetDeploymentBuildEnvVarsResponseBody, GetDeploymentResponseBody, GetEnvironmentVariablesResponseBody, GetLatestDeploymentResponseBody, @@ -689,6 +690,20 @@ export class CliApiClient { ); } + async getDeploymentBuildEnvVars(deploymentId: string) { + if (!this.accessToken) { + throw new Error("getDeploymentBuildEnvVars: No access token"); + } + + return wrapZodFetch( + GetDeploymentBuildEnvVarsResponseBody, + `${this.apiURL}/api/v1/deployments/${deploymentId}/build-env-vars`, + { + headers: this.getHeaders(), + } + ); + } + async getCliPlatformNotification(projectRef?: string, signal?: AbortSignal) { if (!this.accessToken) { return { success: true as const, data: { notification: null } }; diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index 7afa06982ae..74052486c13 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -12,7 +12,7 @@ import type { DeploymentFinalizedEvent, DeploymentTriggeredVia, } from "@trigger.dev/core/v3/schemas"; -import { DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; +import { BuildManifest, DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; import type { Command } from "commander"; import { Option as CommandOption } from "commander"; import { join, relative, resolve } from "node:path"; @@ -24,6 +24,7 @@ import type { CliApiClient } from "../apiClient.js"; import { buildWorker } from "../build/buildWorker.js"; import { resolveAlwaysExternal } from "../build/externals.js"; import { createContextArchive, getArchiveSize } from "../deploy/archiveContext.js"; +import { createBundleArchive } from "../deploy/bundleArchive.js"; import { S2 } from "@s2-dev/streamstore"; import { mkdir, readFile, unlink } from "node:fs/promises"; import { @@ -90,6 +91,8 @@ const DeployCommandOptions = CommonCommandOptions.extend({ push: z.boolean().optional(), builder: z.string().default("trigger"), nativeBuildServer: z.boolean().default(false), + localBundle: z.boolean().default(false), + fromBundle: z.string().optional(), detach: z.boolean().default(false), plain: z.boolean().default(false), compression: z.enum(["zstd", "gzip"]).default("zstd"), @@ -248,6 +251,23 @@ export function configureDeployCommand(program: Command) { "Use the native build server for building the image" ) ) + .addOption( + new CommandOption( + "--local-bundle", + "Experimental: install and bundle locally, upload only the build output, and build the image remotely. Implies using the native build server." + ) + .implies({ nativeBuildServer: true }) + .conflicts(["localBuild", "forceLocalBuild"]) + ) + .addOption( + new CommandOption( + "--from-bundle ", + "Internal: build the image from a pre-built bundle directory. Implies a local build." + ) + .implies({ localBuild: true }) + .conflicts(["nativeBuildServer", "localBundle"]) + .hideHelp() + ) .addOption( new CommandOption( "--detach", @@ -335,6 +355,18 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { logger.debug("Using project ref from env", { ref: envVars.TRIGGER_PROJECT_REF }); } + if (options.fromBundle) { + await handleFromBundleDeploy({ + bundleDir: options.fromBundle, + options, + dashboardUrl: authorization.dashboardUrl, + auth: authorization.auth, + existingDeploymentId: envVars.TRIGGER_EXISTING_DEPLOYMENT_ID, + projectRefOverride: options.projectRef ?? envVars.TRIGGER_PROJECT_REF, + }); + return; + } + let resolvedConfig = await loadConfig({ cwd: projectPath, overrides: { project: options.projectRef ?? envVars.TRIGGER_PROJECT_REF }, @@ -413,6 +445,19 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { resolvedConfig.runtime = projectClient.defaultRuntime; } + if (options.localBundle) { + await handleLocalBundleDeploy({ + apiClient: projectClient.client, + config: resolvedConfig, + dashboardUrl: authorization.dashboardUrl, + options, + userId: userIdForDeploy(authorization), + gitMeta, + branch, + }); + return; + } + if (options.nativeBuildServer) { await handleNativeBuildServerDeploy({ apiClient: projectClient.client, @@ -1635,3 +1680,993 @@ export function verifyDirectory(dir: string, projectPath: string) { throw new Error(`Directory "${dir}" not found at ${projectPath}`); } } + +// --local-bundle: install + bundling happen locally, only the build output is +// uploaded, and the build server runs just the container build from it. +async function handleLocalBundleDeploy({ + apiClient, + options, + config, + dashboardUrl, + userId, + gitMeta, + branch, +}: { + apiClient: CliApiClient; + config: Awaited>; + dashboardUrl: string; + options: DeployCommandOptions; + userId?: string; + gitMeta?: GitMeta; + branch?: string; +}) { + const tmpDir = join(config.workingDir, ".trigger", "tmp"); + await mkdir(tmpDir, { recursive: true }); + + const archivePath = join(tmpDir, `deploy-${Date.now()}.tar.gz`); + + const ignoredBuildFlags = [ + options.compression !== "zstd" && "--compression", + options.cacheCompression !== "zstd" && "--cache-compression", + options.compressionLevel !== undefined && "--compression-level", + !options.forceCompression && "--no-force-compression", + !options.cache && "--no-cache", + options.builder !== "trigger" && "--builder", + options.network !== undefined && "--network", + options.push !== undefined && "--push/--no-push", + options.load !== undefined && "--load/--no-load", + ].filter((flag): flag is string => Boolean(flag)); + + if (ignoredBuildFlags.length > 0) { + log.warn( + `The following flags are ignored with --local-bundle (the image is built remotely): ${ignoredBuildFlags.join(", ")}` + ); + } + + const serverEnvVars = await apiClient.getEnvironmentVariables(config.project); + loadDotEnvVars(config.workingDir, options.envFile); + + // Keep the bundle dir around on dry runs so the printed path is inspectable + const destination = getTmpDir(config.workingDir, "build", options.dryRun); + const forcedExternals = await resolveAlwaysExternal(apiClient); + + const $buildSpinner = spinner({ plain: options.plain }); + + const [buildError, buildManifest] = await tryCatch( + buildWorker({ + target: "deploy", + environment: options.env, + branch, + destination: destination.path, + resolvedConfig: config, + rewritePaths: true, + envVars: serverEnvVars.success ? serverEnvVars.data.variables : {}, + forcedExternals, + plain: options.plain, + listener: { + onBundleStart() { + $buildSpinner.start("Building trigger code"); + }, + onBundleComplete(result) { + $buildSpinner.stop("Successfully built code"); + logger.debug("Bundle result", result); + }, + }, + }) + ); + + if (buildError) { + $buildSpinner.stop("Failed to build code"); + throw buildError; + } + + const bundleManifest = buildManifest; + const bundleOutputPath = destination.path; + + // Extensions can set undefined values at runtime despite the manifest type + const bundleBuildEnvVars = Object.fromEntries( + Object.entries(buildManifest.build.env ?? {}).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + + if (options.dryRun) { + logger.info(`Dry run complete. View the built bundle at ${destination.path}`); + return; + } + + // Sync BEFORE init: init enqueues the build synchronously, so a post-init sync races a fast build + const childVars = buildManifest.deploy.sync?.env ?? {}; + const parentVars = buildManifest.deploy.sync?.parentEnv ?? {}; + const secretChildVars = buildManifest.deploy.sync?.secretEnv ?? {}; + const secretParentVars = buildManifest.deploy.sync?.secretParentEnv ?? {}; + + const hasVarsToSync = + Object.keys(childVars).length > 0 || + Object.keys(secretChildVars).length > 0 || + // Only sync parent variables if this is a branch environment + (branch && (Object.keys(parentVars).length > 0 || Object.keys(secretParentVars).length > 0)); + + if (!options.skipSyncEnvVars) { + if (hasVarsToSync) { + const uploadResult = await syncEnvVarsWithServer( + apiClient, + config.project, + options.env, + childVars, + parentVars, + secretChildVars, + secretParentVars + ); + + if (!uploadResult.success) { + throw new Error(`Failed to sync env vars with the server: ${uploadResult.error}`); + } + + logger.debug("Synced env vars with the server"); + } + } else if (hasVarsToSync) { + logger.log( + "Skipping syncing env vars. The environment variables in your project have changed, but the --skip-sync-env-vars flag was provided." + ); + } + + const $deploymentSpinner = spinner(); + $deploymentSpinner.start("Preparing deployment files"); + + await createBundleArchive(bundleOutputPath, archivePath); + + const archiveSize = await getArchiveSize(archivePath); + const sizeMB = (archiveSize / 1024 / 1024).toFixed(2); + $deploymentSpinner.message(`Deployment files ready (${sizeMB} MB)`); + + const artifactResult = await apiClient.createArtifact({ + type: "deployment_bundle", + contentType: "application/gzip", + contentLength: archiveSize, + }); + + if (!artifactResult.success) { + $deploymentSpinner.stop("Failed creating deployment artifact"); + log.error(chalk.bold(chalkError(artifactResult.error))); + throw new OutroCommandError(`Deployment failed`); + } + + const { artifactKey, uploadUrl, uploadFields } = artifactResult.data; + + logger.debug("Artifact created", { artifactKey }); + + // Defense in depth: current older servers already reject the deployment_bundle + // type at createArtifact; this catches a server that accepts it but returns a + // non-bundle key, which would make the remote build treat the bundle as source. + if (!artifactKey.startsWith("bundles/")) { + $deploymentSpinner.stop("Failed creating deployment artifact"); + log.error( + chalk.bold( + chalkError( + "This server does not support --local-bundle deploys yet. Deploy without --local-bundle instead." + ) + ) + ); + throw new OutroCommandError(`Deployment failed`); + } + + $deploymentSpinner.message("Uploading deployment files"); + + const [readError, fileBuffer] = await tryCatch(readFile(archivePath)); + + if (readError) { + $deploymentSpinner.stop("Failed reading deployment archive"); + log.error(chalk.bold(chalkError(readError.message))); + throw new OutroCommandError(`Deployment failed`); + } + + const formData = new FormData(); + + for (const [key, value] of Object.entries(uploadFields)) { + formData.append(key, value); + } + + const blob = new Blob([new Uint8Array(fileBuffer)], { type: "application/gzip" }); + formData.append("file", blob, "deployment.tar.gz"); + + const [uploadError, uploadResponse] = await tryCatch( + fetch(uploadUrl, { + method: "POST", + body: formData, + }) + ); + + if (uploadError || !uploadResponse?.ok) { + $deploymentSpinner.stop("Failed to upload deployment files"); + log.error( + chalk.bold( + chalkError( + `${uploadError?.message} (${uploadResponse?.statusText} ${uploadResponse?.status})` + ) + ) + ); + throw new OutroCommandError(`Deployment failed`); + } + + const [unlinkError] = await tryCatch(unlink(archivePath)); + if (unlinkError) { + logger.debug("Failed to delete deployment artifact file", { archivePath, error: unlinkError }); + } + + $deploymentSpinner.message("Deployment files uploaded"); + + const configFilePath = + config.configFile !== undefined + ? relative(config.workspaceDir, config.configFile).replace(/\\/g, "/") + : undefined; + + const initializeDeploymentResult = await apiClient.initializeDeployment({ + contentHash: bundleManifest.contentHash, + userId, + gitMeta, + type: config.features.run_engine_v2 ? "MANAGED" : "V1", + // config.runtime (not the manifest runtime) to match classic native deploys + runtime: config.runtime, + isNativeBuild: true, + artifactKey, + skipPromotion: options.skipPromotion, + configFilePath, + triggeredVia: getTriggeredVia(), + externalId: options.externalId, + force: options.force, + fromBundle: true, + buildEnvVars: Object.keys(bundleBuildEnvVars).length > 0 ? bundleBuildEnvVars : undefined, + }); + + if (!initializeDeploymentResult.success) { + $deploymentSpinner.stop("Failed to initialize deployment"); + log.error(chalk.bold(chalkError(initializeDeploymentResult.error))); + throw new OutroCommandError(`Deployment failed`); + } + + const deployment = initializeDeploymentResult.data; + + const rawDeploymentLink = `${dashboardUrl}/projects/v3/${config.project}/deployments/${deployment.shortCode}`; + const rawTestLink = `${dashboardUrl}/projects/v3/${config.project}/test?environment=${ + options.env === "prod" ? "prod" : "stg" + }`; + + if (deployment.outcome === "existing") { + $deploymentSpinner.stop(`Version ${deployment.version} was already deployed`); + + setDeploymentGithubActionsOutput({ + version: deployment.version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: !deployment.isPromoted, + }); + + warnAboutSkippedBuild(options.externalId, deployment.isPromoted); + + outro( + `Version ${deployment.version} was already deployed for --external-id ${options.externalId} — nothing to build ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : rawDeploymentLink + }` + ); + + return; + } + + const exposedDeploymentLink = isLinksSupported + ? cliLink(chalk.bold(rawDeploymentLink), rawDeploymentLink) + : chalk.bold(rawDeploymentLink); + $deploymentSpinner.stop("Deployment initialized"); + log.info(`View deployment: ${exposedDeploymentLink}`); + + warnAboutCanceledDeployments(deployment.canceledDeployments, options.externalId); + + setDeploymentGithubActionsOutput({ + version: deployment.version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: options.skipPromotion, + }); + + if (options.detach) { + outro(`Version ${deployment.version} is being deployed`); + return; + } + + const { eventStream } = deployment; + + if (!eventStream) { + log.warn(`Failed streaming build logs, open the deployment in the dashboard to view the logs`); + + outro(`Version ${deployment.version} is being deployed`); + + return process.exit(0); + } + + const $queuedSpinner = spinner(); + $queuedSpinner.start("Build queued"); + + const abortController = new AbortController(); + + const s2 = new S2({ accessToken: eventStream.s2.accessToken }); + const basin = s2.basin(eventStream.s2.basin); + const stream = basin.stream(eventStream.s2.stream); + + const [readSessionError, readSession] = await tryCatch( + stream.readSession( + { + start: { from: { seqNum: 0 }, clamp: true }, + stop: { waitSecs: 60 * 20 }, // 20 minutes + }, + { signal: abortController.signal } + ) + ); + + if (readSessionError) { + $queuedSpinner.stop("Failed to query build progress"); + log.warn(`Failed streaming build logs, open the deployment in the dashboard to view the logs`); + + outro( + `Version ${deployment.version} is being deployed ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + + return process.exit(0); + } + + let finalDeploymentEvent: DeploymentFinalizedEvent["data"] | undefined; + let queuedSpinnerStopped = false; + + for await (const record of readSession) { + const decoded = record.body; + const result = DeploymentEventFromString.safeParse(decoded); + if (!result.success) { + logger.debug("Failed to parse deployment event, skipping", { + error: result.error, + record: decoded, + }); + continue; + } + + const event = result.data; + + switch (event.type) { + case "log": { + if (record.seqNum === 0) { + $queuedSpinner.stop("Build started"); + console.log("│"); + queuedSpinnerStopped = true; + } + + const formattedTimestamp = chalkGrey( + new Date(record.timestamp).toLocaleTimeString("en-US", { + hour12: false, + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + fractionalSecondDigits: 3, + }) + ); + + const { level, message } = event.data; + const formattedMessage = + level === "error" + ? chalk.bold(chalkError(message)) + : level === "warn" + ? chalkWarning(message) + : level === "debug" + ? chalkGrey(message) + : message; + + // We use console.log here instead of clack's logger as the current version does not support changing the line spacing. + // And the logs look verbose with the default spacing. + // We cannot upgrade because the newer versions introduced some weird issues with the spinner. + // Ideally, we'd use clack's `taskLog` to only show the recent n lines of logs as they are streamed, but that also seems brittle + // and has some issues with cursor movements/clearing lines that it shouldn't clear. + // We can revisit this on future versions of `@clack/prompts`. + console.log(`│ ${formattedTimestamp} ${formattedMessage}`); + break; + } + case "finalized": { + finalDeploymentEvent = event.data; + abortController.abort(); // stop the stream + break; + } + default: { + event satisfies never; + logger.debug("Unknown deployment event, skipping", { event }); + continue; + } + } + } + + if (!queuedSpinnerStopped && !finalDeploymentEvent) { + // unlikely that it happens in practice, only in rare corner cases + // the timeout would kick in earlier if the build server fails to dequeue the build + + $queuedSpinner.stop("Log stream stopped"); + + log.error("Failed dequeueing build, please try again shortly"); + + throw new OutroCommandError( + `Version ${deployment.version} ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + + if (!finalDeploymentEvent) { + log.error( + "Stopped receiving updates from the build server, please check the deployment status in the dashboard" + ); + + if (!isLinksSupported) { + log.info(`View deployment: ${rawDeploymentLink}`); + } + + throw new OutroCommandError( + `Version ${deployment.version} ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + + switch (finalDeploymentEvent.result) { + case "succeeded": { + queuedSpinnerStopped + ? log.success("Deployment completed successfully") + : $queuedSpinner.stop("Deployment completed successfully"); + + if (finalDeploymentEvent.message) { + log.success(finalDeploymentEvent.message); + } + + if (options.skipPromotion) { + log.info( + `This deployment was not automatically promoted. You can promote in the dashboard or via the promote command, e.g, \`npx trigger.dev promote ${deployment.version}\`.` + ); + } + + if (!isLinksSupported) { + log.info(`Test tasks: ${rawTestLink}`); + } + + outro( + `Version ${deployment.version} was deployed ${ + isLinksSupported + ? `| ${cliLink("Test tasks", rawTestLink)} | ${cliLink( + "View deployment", + rawDeploymentLink + )}` + : "" + }` + ); + return process.exit(0); + } + case "failed": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment failed"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment failed" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment failed ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + case "timed_out": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment timed out"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment timed out" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment timed out ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + case "canceled": { + if (!queuedSpinnerStopped) { + $queuedSpinner.stop("Deployment was canceled"); + } + + log.error( + chalk.bold( + chalkError( + "Deployment was canceled" + + (finalDeploymentEvent.message ? `: ${finalDeploymentEvent.message}` : "") + ) + ) + ); + + throw new OutroCommandError( + `Version ${deployment.version} deployment canceled ${ + isLinksSupported ? `| ${cliLink("View deployment", rawDeploymentLink)}` : "" + }` + ); + } + default: { + // This case is only relevant in case we extend the enum in the future. + // New enum values will not be treated as errors in older cli versions. + queuedSpinnerStopped + ? log.success("Log stream finished") + : $queuedSpinner.stop("Log stream finished"); + if (finalDeploymentEvent.message) { + log.message(finalDeploymentEvent.message); + } + + if (!isLinksSupported) { + log.info(`Test tasks: ${rawTestLink}`); + } + + outro( + `Version ${deployment.version} ${ + isLinksSupported + ? `| ${cliLink("Test tasks", rawTestLink)} | ${cliLink( + "View deployment", + rawDeploymentLink + )}` + : "" + }` + ); + return process.exit(0); + } + } +} + +// Builds the image locally from the bundle and finalizes the deployment. +async function buildAndFinalizeFromBundle({ + apiClient, + projectId, + projectRef, + deployment, + options, + dashboardUrl, + authAccessToken, + compilationPath, + buildEnvVars, + branch, + isLocalBuild, +}: { + apiClient: CliApiClient; + projectId: string; + projectRef: string; + deployment: Deployment; + options: DeployCommandOptions; + dashboardUrl: string; + authAccessToken: string; + compilationPath: string; + buildEnvVars: Record | undefined; + branch: string | undefined; + isLocalBuild: boolean; +}) { + const authenticateToTriggerRegistry = options.localBuild; + const skipServerSideRegistryPush = options.localBuild; + + const version = deployment.version; + + const { rawDeploymentLink, rawTestLink } = buildDeploymentLinks({ + dashboardUrl, + projectRef, + env: options.env, + shortCode: deployment.shortCode, + }); + + const deploymentLink = cliLink("View deployment", rawDeploymentLink); + const testLink = cliLink("Test tasks", rawTestLink); + + const $spinner = spinner({ plain: options.plain }); + + const buildSuffix = + isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_LABEL_DISABLED !== "1" ? " (local)" : ""; + const deploySuffix = + isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_LABEL_DISABLED !== "1" ? " (local build)" : ""; + + if (options.plain) { + $spinner.start(`Building version ${version}${buildSuffix}`); + } else if (isCI) { + log.step(`Building version ${version}\n`); + } else { + if (isLinksSupported) { + $spinner.start(`Building version ${version}${buildSuffix} ${deploymentLink}`); + } else { + $spinner.start(`Building version ${version}${buildSuffix}`); + } + } + + const buildResult = await buildImage({ + isLocalBuild, + useRegistryCache: options.useRegistryCache, + noCache: !options.cache, + deploymentId: deployment.id, + deploymentVersion: deployment.version, + imageTag: deployment.imageTag, + imagePlatform: deployment.imagePlatform, + load: options.load, + contentHash: deployment.contentHash, + externalBuildId: deployment.externalBuildData?.buildId, + externalBuildToken: deployment.externalBuildData?.buildToken, + externalBuildProjectId: deployment.externalBuildData?.projectId, + projectId, + projectRef, + apiUrl: apiClient.apiURL, + apiKey: apiClient.accessToken!, + apiClient, + branchName: branch, + authAccessToken, + compilationPath, + buildEnvVars, + compression: options.compression, + cacheCompression: options.cacheCompression, + compressionLevel: options.compressionLevel, + forceCompression: options.forceCompression, + onLog: (logMessage) => { + if (options.plain || isCI) { + console.log(logMessage); + return; + } + + if (isLinksSupported) { + $spinner.message( + `Building version ${version}${buildSuffix} ${deploymentLink}: ${logMessage}` + ); + } else { + $spinner.message(`Building version ${version}${buildSuffix}: ${logMessage}`); + } + }, + // Local build options + network: options.network, + builder: options.builder, + push: options.push, + authenticateToRegistry: authenticateToTriggerRegistry, + }); + + logger.debug("Build result", buildResult); + + const warnings = checkLogsForWarnings(buildResult.logs); + + const canShowLocalBuildHint = + !isLocalBuild && process.env.TRIGGER_LOCAL_BUILD_HINT_DISABLED !== "1"; + const buildFailed = !warnings.ok || !buildResult.ok; + + if (buildFailed && canShowLocalBuildHint) { + const providerStatus = await apiClient.getRemoteBuildProviderStatus(); + + if (providerStatus.success && providerStatus.data.status === "degraded") { + prettyWarning(providerStatus.data.message + "\n"); + } + } + + if (!warnings.ok) { + await failDeploy( + apiClient, + deployment, + { name: "BuildError", message: warnings.summary }, + buildResult.logs, + $spinner, + warnings.warnings, + warnings.errors + ); + + throw new SkipLoggingError("Failed to build image"); + } + + if (!buildResult.ok) { + await failDeploy( + apiClient, + deployment, + { name: "BuildError", message: buildResult.error }, + buildResult.logs, + $spinner, + warnings.warnings + ); + + throw new SkipLoggingError("Failed to build image"); + } + + const getDeploymentResponse = await apiClient.getDeployment(deployment.id); + + if (!getDeploymentResponse.success) { + await failDeploy( + apiClient, + deployment, + { name: "DeploymentError", message: getDeploymentResponse.error }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError(getDeploymentResponse.error); + } + + const deploymentWithWorker = getDeploymentResponse.data; + + if (!deploymentWithWorker.worker) { + const errorData = deploymentWithWorker.errorData + ? prepareDeploymentError(deploymentWithWorker.errorData) + : undefined; + + await failDeploy( + apiClient, + deployment, + { + name: "DeploymentError", + message: errorData?.message ?? "Failed to get deployment with worker", + }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError(errorData?.message ?? "Failed to get deployment with worker"); + } + + if (options.plain) { + $spinner.message(`Deploying version ${version}${deploySuffix}`); + } else if (isCI) { + log.step(`Deploying version ${version}${deploySuffix}\n`); + } else { + if (isLinksSupported) { + $spinner.message(`Deploying version ${version}${deploySuffix} ${deploymentLink}`); + } else { + $spinner.message(`Deploying version ${version}${deploySuffix}`); + } + } + + const finalizeResponse = await apiClient.finalizeDeployment( + deployment.id, + { + imageDigest: buildResult.digest, + skipPromotion: options.skipPromotion, + skipPushToRegistry: skipServerSideRegistryPush, + }, + (logMessage) => { + if (options.plain || isCI) { + console.log(logMessage); + return; + } + + if (isLinksSupported) { + $spinner.message( + `Deploying version ${version}${deploySuffix} ${deploymentLink}: ${logMessage}` + ); + } else { + $spinner.message(`Deploying version ${version}${deploySuffix}: ${logMessage}`); + } + } + ); + + if (!finalizeResponse.success) { + await failDeploy( + apiClient, + deployment, + { name: "FinalizeError", message: finalizeResponse.error }, + buildResult.logs, + $spinner + ); + + throw new SkipLoggingError("Failed to finalize deployment"); + } + + if (options.plain) { + console.log(`Successfully deployed version ${version}${deploySuffix}`); + } else if (isCI) { + log.step(`Successfully deployed version ${version}${deploySuffix}`); + } else { + $spinner.stop(`Successfully deployed version ${version}${deploySuffix}`); + } + + const taskCount = deploymentWithWorker.worker?.tasks.length ?? 0; + + if (options.plain) { + console.log( + `Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s"}` + ); + + if (process.env.TRIGGER_DEPLOYMENT_LINK_OUTPUT_DISABLED !== "1") { + console.log(`Deployment: ${rawDeploymentLink}`); + console.log(`Test: ${rawTestLink}`); + } + } else { + outro( + `Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s"} ${ + isLinksSupported ? `| ${deploymentLink} | ${testLink}` : "" + }` + ); + + if (!isLinksSupported) { + console.log("View deployment"); + console.log(rawDeploymentLink); + console.log(); // new line + console.log("Test tasks"); + console.log(rawTestLink); + } + } + + if (options.saveLogs) { + const logPath = await saveLogs(deployment.shortCode, buildResult.logs); + console.log(`Full build logs have been saved to ${logPath}`); + } + + setDeploymentGithubActionsOutput({ + version, + shortCode: deployment.shortCode, + rawDeploymentLink, + rawTestLink, + needsPromotion: options.skipPromotion, + }); +} + +// Runs only the container build from a pre-built bundle dir, skipping config loading +// entirely. Attach mode is the supported flow (build server); fresh-init is for testing. +async function handleFromBundleDeploy({ + bundleDir, + options, + dashboardUrl, + auth, + existingDeploymentId, + projectRefOverride, +}: { + bundleDir: string; + options: DeployCommandOptions; + dashboardUrl: string; + auth: { accessToken: string; apiUrl: string }; + existingDeploymentId?: string; + projectRefOverride?: string; +}) { + const bundlePath = resolve(process.cwd(), bundleDir); + + if (!isDirectory(bundlePath)) { + throw new Error(`Bundle directory not found at ${bundlePath}`); + } + + const [manifestReadError, manifestRaw] = await tryCatch( + readFile(join(bundlePath, "build.json"), "utf-8") + ); + + if (manifestReadError) { + throw new Error( + `Failed to read build.json in the bundle directory: ${manifestReadError.message}` + ); + } + + let manifestJson: unknown; + try { + manifestJson = JSON.parse(manifestRaw); + } catch { + throw new Error(`Invalid build.json in the bundle directory: not valid JSON`); + } + + const manifestResult = BuildManifest.safeParse(manifestJson); + + if (!manifestResult.success) { + throw new Error(`Invalid build.json in the bundle directory: ${manifestResult.error.message}`); + } + + const bundleManifest = manifestResult.data; + + // --dry-run must never touch the server + if (options.dryRun) { + logger.info(`Dry run complete. Validated bundle at ${bundlePath}`); + return; + } + + const projectRef = projectRefOverride ?? bundleManifest.config.project; + + const branch = options.env === "preview" ? getBranch({ specified: options.branch }) : undefined; + + if (options.env === "preview" && !branch) { + throw new Error( + "Preview deploys from a bundle require an explicit branch. Pass --branch ." + ); + } + + // In attach mode the branch env already exists + if (options.env === "preview" && branch && !existingDeploymentId) { + await upsertBranch({ + accessToken: auth.accessToken, + apiUrl: auth.apiUrl, + projectRef, + branch, + gitMeta: undefined, + }); + } + + const projectClient = await getProjectClient({ + accessToken: auth.accessToken, + apiUrl: auth.apiUrl, + projectRef, + env: options.env, + branch, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + // In attach mode the build-arg values are stored encrypted on the deployment + let buildEnvVars: Record | undefined; + + if (existingDeploymentId) { + const buildEnvVarsResult = + await projectClient.client.getDeploymentBuildEnvVars(existingDeploymentId); + + if (!buildEnvVarsResult.success) { + throw new Error( + `Failed to fetch the build environment variables for deployment ${existingDeploymentId}: ${buildEnvVarsResult.error}` + ); + } + + buildEnvVars = buildEnvVarsResult.data.variables; + } else if (bundleManifest.build.env && Object.keys(bundleManifest.build.env).length > 0) { + // Extensions can set undefined values at runtime despite the manifest type + buildEnvVars = Object.fromEntries( + Object.entries(bundleManifest.build.env).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + } + + if (!existingDeploymentId) { + logger.warn( + "No existing deployment to attach to — initializing a fresh local-build deployment from the bundle. This path is intended for testing." + ); + } + + const deployment = await initializeOrAttachDeployment( + projectClient.client, + { + contentHash: bundleManifest.contentHash, + type: "MANAGED", + runtime: bundleManifest.runtime, + isLocalBuild: true, + isNativeBuild: false, + triggeredVia: getTriggeredVia(), + }, + existingDeploymentId + ); + + // Fail fast if we know local builds will fail + const buildxResult = await x("docker", ["buildx", "version"]); + + if (buildxResult.exitCode !== 0) { + logger.debug(`"docker buildx version" failed (${buildxResult.exitCode}):`, buildxResult); + throw new Error( + "Failed to find docker buildx. Please install it: https://github.com/docker/buildx#installing." + ); + } + + await buildAndFinalizeFromBundle({ + apiClient: projectClient.client, + projectId: projectClient.id, + projectRef, + deployment, + options, + dashboardUrl, + authAccessToken: auth.accessToken, + compilationPath: bundlePath, + buildEnvVars, + branch, + isLocalBuild: true, + }); +} diff --git a/packages/cli-v3/src/deploy/bundleArchive.test.ts b/packages/cli-v3/src/deploy/bundleArchive.test.ts new file mode 100644 index 00000000000..efd79b5aefd --- /dev/null +++ b/packages/cli-v3/src/deploy/bundleArchive.test.ts @@ -0,0 +1,106 @@ +import { mkdtemp, mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as tar from "tar"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createBundleArchive } from "./bundleArchive.js"; + +describe("createBundleArchive", () => { + let bundleDir: string; + let outDir: string; + + beforeEach(async () => { + bundleDir = await mkdtemp(join(tmpdir(), "bundle-src-")); + outDir = await mkdtemp(join(tmpdir(), "bundle-out-")); + }); + + afterEach(async () => { + await rm(bundleDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); + }); + + it("archives bundle contents at the root, including dotfiles and nested dirs", async () => { + await writeFile(join(bundleDir, "build.json"), JSON.stringify({ contentHash: "abc" })); + await writeFile(join(bundleDir, "Containerfile"), "FROM scratch"); + await writeFile(join(bundleDir, "package.json"), "{}"); + await writeFile(join(bundleDir, "index.mjs"), "export {}"); + await writeFile(join(bundleDir, ".dockerignore"), "*.log\n"); + await mkdir(join(bundleDir, ".trigger", "skills", "my-skill"), { recursive: true }); + await writeFile(join(bundleDir, ".trigger", "skills", "my-skill", "SKILL.md"), "# skill"); + + const archivePath = join(outDir, "bundle.tar.gz"); + await createBundleArchive(bundleDir, archivePath); + + const extractDir = join(outDir, "extracted"); + await mkdir(extractDir); + await tar.extract({ file: archivePath, cwd: extractDir }); + + const rootEntries = (await readdir(extractDir)).sort(); + expect(rootEntries).toEqual( + [ + ".dockerignore", + ".trigger", + "Containerfile", + "build.json", + "index.mjs", + "package.json", + ].sort() + ); + + const skill = await readFile( + join(extractDir, ".trigger", "skills", "my-skill", "SKILL.md"), + "utf-8" + ); + expect(skill).toBe("# skill"); + }); + + it("excludes only .DS_Store — node_modules paths must survive", async () => { + await writeFile(join(bundleDir, "build.json"), "{}"); + await writeFile(join(bundleDir, ".DS_Store"), "junk"); + // Under npx the controller entry points live beneath a node_modules segment + const controllerDir = join( + bundleDir, + ".npm", + "_npx", + "abc123", + "node_modules", + "trigger.dev", + "dist" + ); + await mkdir(controllerDir, { recursive: true }); + await writeFile(join(controllerDir, "managed-index-controller.mjs"), "x"); + await mkdir(join(bundleDir, "dist"), { recursive: true }); + await writeFile(join(bundleDir, "dist", "chunk.mjs"), "x"); + + const archivePath = join(outDir, "bundle.tar.gz"); + await createBundleArchive(bundleDir, archivePath); + + const extractDir = join(outDir, "extracted"); + await mkdir(extractDir); + await tar.extract({ file: archivePath, cwd: extractDir }); + + const rootEntries = (await readdir(extractDir)).sort(); + expect(rootEntries).toEqual(["build.json", "dist", ".npm"].sort()); + + const controller = await readFile( + join( + extractDir, + ".npm", + "_npx", + "abc123", + "node_modules", + "trigger.dev", + "dist", + "managed-index-controller.mjs" + ), + "utf-8" + ); + expect(controller).toBe("x"); + }); + + it("throws when the bundle dir is empty", async () => { + await expect(createBundleArchive(bundleDir, join(outDir, "bundle.tar.gz"))).rejects.toThrow( + /No files found/ + ); + }); +}); diff --git a/packages/cli-v3/src/deploy/bundleArchive.ts b/packages/cli-v3/src/deploy/bundleArchive.ts new file mode 100644 index 00000000000..f53c820df89 --- /dev/null +++ b/packages/cli-v3/src/deploy/bundleArchive.ts @@ -0,0 +1,40 @@ +import { glob } from "tinyglobby"; +import * as tar from "tar"; +import { logger } from "../utilities/logger.js"; + +// The bundle dir is generated build output, so the usual source ignores (dist, +// node_modules, ...) would strip load-bearing files: under npx the controller +// entry points live beneath a node_modules path segment. +const BUNDLE_IGNORES = ["**/.DS_Store"]; + +// Bundle contents land at the archive root; the build server extracts without stripping +export async function createBundleArchive(bundleDir: string, outputPath: string) { + logger.debug("Creating bundle archive", { bundleDir, outputPath }); + + const files = await glob(["**/*"], { + cwd: bundleDir, + ignore: BUNDLE_IGNORES, + dot: true, // .trigger/skills and .dockerignore must be included + absolute: false, + onlyFiles: true, + followSymbolicLinks: false, + }); + + if (files.length === 0) { + throw new Error("No files found in the bundle output. This is likely a bug."); + } + + await tar.create( + { + gzip: true, + file: outputPath, + cwd: bundleDir, + portable: true, + preservePaths: false, + mtime: new Date(0), + }, + files + ); + + logger.debug("Bundle archive created", { outputPath, fileCount: files.length }); +} diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 08dc4d9ca0a..a90430953d4 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -654,6 +654,7 @@ export const BuildServerMetadata = z.object({ skipPromotion: z.boolean().optional(), configFilePath: z.string().optional(), skipEnqueue: z.boolean().optional(), + fromBundle: z.boolean().optional(), }); export type BuildServerMetadata = z.infer; @@ -720,7 +721,7 @@ export const UpsertBranchResponseBody = z.object({ export type UpsertBranchResponseBody = z.infer; export const CreateArtifactRequestBody = z.object({ - type: z.enum(["deployment_context"]).default("deployment_context"), + type: z.enum(["deployment_context", "deployment_bundle"]).default("deployment_context"), contentType: z.string().default("application/gzip"), contentLength: z.number().optional(), }); @@ -784,6 +785,8 @@ type NativeBuildOutput = BaseOutput & { artifactKey?: string; configFilePath?: string; skipEnqueue?: boolean; + fromBundle?: boolean; + buildEnvVars?: Record; }; type NonNativeBuildOutput = BaseOutput & { @@ -792,6 +795,8 @@ type NonNativeBuildOutput = BaseOutput & { artifactKey?: never; configFilePath?: never; skipEnqueue?: never; + fromBundle?: never; + buildEnvVars?: never; }; const InitializeDeploymentRequestBodyFull = InitializeDeploymentRequestBodyBase.extend({ @@ -800,6 +805,10 @@ const InitializeDeploymentRequestBodyFull = InitializeDeploymentRequestBodyBase. artifactKey: z.string().optional(), configFilePath: z.string().optional(), skipEnqueue: z.boolean().optional().default(false), + // The artifact is a pre-built bundle; the build server only runs the container build + fromBundle: z.boolean().optional(), + // Build-time env var values for fromBundle deploys, stored encrypted on the deployment + buildEnvVars: z.record(z.string()).optional(), }).superRefine((data, ctx) => { if (data.force && !data.externalId) { ctx.addIssue({ @@ -815,7 +824,15 @@ export const InitializeDeploymentRequestBody = InitializeDeploymentRequestBodyFu if (data.isNativeBuild) { return { ...data, isNativeBuild: true as const }; } - const { skipPromotion, artifactKey, configFilePath, skipEnqueue, ...rest } = data; + const { + skipPromotion, + artifactKey, + configFilePath, + skipEnqueue, + fromBundle, + buildEnvVars, + ...rest + } = data; return { ...rest, isNativeBuild: false as const }; } ); @@ -921,6 +938,15 @@ export const GetDeploymentResponseBody = z.object({ export type GetDeploymentResponseBody = z.infer; +// Secret material, deliberately kept off GetDeploymentResponseBody +export const GetDeploymentBuildEnvVarsResponseBody = z.object({ + variables: z.record(z.string()), +}); + +export type GetDeploymentBuildEnvVarsResponseBody = z.infer< + typeof GetDeploymentBuildEnvVarsResponseBody +>; + export const GetLatestDeploymentResponseBody = GetDeploymentResponseBody.omit({ worker: true, }); From 2e87e93934ad2ffb8889f70b821cd5d862547e62 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:18:45 +0100 Subject: [PATCH 08/28] ci: run codeql on all prs via advanced setup (#4767) Default setup doesn't run CodeQL on pull requests from forks, so external contributions are stuck on PR checks that never come. Advanced setup fixes this. Languages, categories and `main` coverage match the current default setup. The bare `pull_request` trigger (no `branches` filter) keeps stacked PRs scanned, whose base isn't `main`. Default setup has to be disabled in Settings -> Code security for these uploads to be accepted. Until it is, the CodeQL check here fails with `CodeQL analyses from advanced configurations cannot be processed when the default setup is enabled`. --- .github/workflows/codeql.yml | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..9fb39b1c505 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,40 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + if: github.repository == 'triggerdotdev/trigger.dev' + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # Upload SARIF to GitHub Security tab + strategy: + fail-fast: false + matrix: + language: [actions, javascript-typescript] + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + category: /language:${{ matrix.language }} From 11e1cd817475f08225ad614c6e9166aa066a608c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 25 Aug 2026 09:19:48 +0100 Subject: [PATCH 09/28] feat(webapp): isolate the runs list ClickHouse read pool (#4763) ## Summary Improves the performance and reliability of the runs list and the `runs.list` API, especially for large projects and filtered views. ## What changed - **Filtered runs-list queries use `PREWHERE`.** Immutable and additive-only filters (tags, task identifier, version, queue, region, machine, and the rest) are applied in `PREWHERE` on the `task_runs_v2 FINAL` scan, so ClickHouse filters, and uses the tags skip index, before it reconciles versions and materialises the wide columns. Same results, far less memory per query. `status` stays in `WHERE`: it changes across a run's versions, so filtering it before `FINAL` could return stale rows. - **The runs-list ClickHouse pool gets per-query guardrails**, all env-configurable: a `max_execution_time` paired with the client request timeout, a per-query `max_memory_usage`, a `max_threads` cap, and `readonly`. Each bounds a single query to itself, so a heavy query can't affect other queries, and they are safe as pool-level settings only because this pool is read-only. - **Billing and bulk count reads move to the read pool**, off the write pool. Defaults are conservative for self-hosters; production values are set via env. --- .server-changes/runs-list-read-isolation.md | 6 + apps/webapp/app/env.server.ts | 9 ++ .../v3/CreateBulkActionPresenter.server.ts | 2 +- .../clickhouse/clickhouseFactory.server.ts | 60 ++++++++- .../clickhouseRunsRepository.server.ts | 88 ++++++++----- .../billingLimitQueuedRuns.server.ts | 4 +- .../v3/services/bulk/BulkActionV2.server.ts | 4 +- .../test/runsListClickhouseSettings.test.ts | 66 ++++++++++ apps/webapp/test/runsListQueryShape.test.ts | 124 ++++++++++++++++++ .../clickhouse/src/client/queryBuilder.ts | 26 ++++ 10 files changed, 346 insertions(+), 43 deletions(-) create mode 100644 .server-changes/runs-list-read-isolation.md create mode 100644 apps/webapp/test/runsListClickhouseSettings.test.ts create mode 100644 apps/webapp/test/runsListQueryShape.test.ts diff --git a/.server-changes/runs-list-read-isolation.md b/.server-changes/runs-list-read-isolation.md new file mode 100644 index 00000000000..5c411635735 --- /dev/null +++ b/.server-changes/runs-list-read-isolation.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Improved the performance and reliability of the runs list and the runs.list API, especially for large projects and filtered views. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 496c6e3d9a6..c63c79d41d6 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2246,6 +2246,15 @@ const EnvironmentSchema = z .enum(["log", "error", "warn", "info", "debug"]) .default("info"), RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), + RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(40_000), + RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().positive().default(35), + RUNS_LIST_CLICKHOUSE_MAX_THREADS: z.coerce.number().int().positive().default(4), + RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce + .number() + .int() + .positive() + .default(1_073_741_824), + RUNS_LIST_CLICKHOUSE_READONLY: z.enum(["0", "1", "2"]).default("2"), /** * Dedicated ClickHouse service for queue metrics: the ingestion consumer's inserts and every * queue-metrics read (dashboards, queue pages, run inspector, health report) go through it, so diff --git a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts index a04368b4255..6ddd6250b58 100644 --- a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts @@ -26,7 +26,7 @@ export class CreateBulkActionPresenter extends BasePresenter { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 5a2b3b86eee..a29664a4502 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -1,4 +1,4 @@ -import { ClickHouse } from "@internal/clickhouse"; +import { ClickHouse, type ClickHouseSettings } from "@internal/clickhouse"; import { createHash } from "crypto"; import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { env } from "~/env.server"; @@ -292,6 +292,45 @@ function initializeRealtimeClickhouseClient(): ClickHouse { }); } +/** + * Server-side query protection for the runs-list read pool. Every setting here is PER-QUERY, so a + * pathological query only ever kills itself: a slow one hits `max_execution_time`, a memory-hungry + * one hits `max_memory_usage`, a thread-hungry one hits `max_threads`. Per-USER limits + * (`max_*_for_user`) are deliberately NOT used: everything connects as `default`, so a per-user cap + * would reject whichever query arrives once the shared budget is hit, punishing innocent tenants + * for a noisy one. The node itself is protected by the server-level `max_server_memory_usage`. + * Safe as client-level settings ONLY because this pool is read-only; on a mixed read+write pool a + * client-level `max_execution_time` would also kill slow inserts. `readonly=2` enforces read-only + * while still allowing these settings to apply (`readonly=1` rejects them). + */ +/** + * Client request timeout for the runs-list pool, forced above the server-side `max_execution_time` + * so the server cap is what stops a slow query and the client stays connected to receive that + * error. If the client timed out first, it would abort while ClickHouse kept executing, which is + * the abandoned-query behaviour this pool is trying to prevent. + */ +function getRunsListRequestTimeoutMs() { + return Math.max( + env.RUNS_LIST_CLICKHOUSE_REQUEST_TIMEOUT_MS, + (env.RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME + 5) * 1000 + ); +} + +function getRunsListClickhouseSettings(): ClickHouseSettings { + const settings: ClickHouseSettings = { + max_execution_time: env.RUNS_LIST_CLICKHOUSE_MAX_EXECUTION_TIME, + timeout_before_checking_execution_speed: 0, + max_threads: env.RUNS_LIST_CLICKHOUSE_MAX_THREADS, + max_memory_usage: env.RUNS_LIST_CLICKHOUSE_MAX_MEMORY_USAGE.toString(), + }; + + if (env.RUNS_LIST_CLICKHOUSE_READONLY !== "0") { + settings.readonly = env.RUNS_LIST_CLICKHOUSE_READONLY; + } + + return settings; +} + /** Runs list reads — dashboard + API (`RUNS_LIST_CLICKHOUSE_URL`); * falls back to the default client if unset. */ const defaultRunsListClickhouseClient = singleton( @@ -319,6 +358,8 @@ function initializeRunsListClickhouseClient(): ClickHouse { request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1", }, maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + requestTimeoutMs: getRunsListRequestTimeoutMs(), + clickhouseSettings: getRunsListClickhouseSettings(), }); } @@ -550,10 +591,25 @@ function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHou }, maxOpenConnections: env.REALTIME_BACKEND_NATIVE_CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); + case "runsList": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.RUNS_LIST_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.RUNS_LIST_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.RUNS_LIST_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + requestTimeoutMs: getRunsListRequestTimeoutMs(), + clickhouseSettings: getRunsListClickhouseSettings(), + }); case "standard": case "query": case "admin": - case "runsList": return new ClickHouse({ url: parsed.toString(), name, diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2e911e5e958..4981dd19c43 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -411,6 +411,22 @@ export class ClickHouseRunsRepository implements IRunsRepository { } } +/** + * Builds the shared filter clauses for the runs list against `task_runs_v2 FINAL`. + * + * A filter may go in PREWHERE only if its truth value can never flip true->false across a run's + * versions, because PREWHERE is evaluated before FINAL reconciles versions and would otherwise keep + * a stale matching version and drop the winning one. That holds for columns that are only ever set + * once and never change: trigger-time identity columns (task_identifier, task_version, schedule_id, + * is_test, root_run_id, batch_id, friendly_id, queue, task_kind), append-only arrays under + * `hasAny`/`hasAll` (tags, bulk_action_group_ids), `region` (set once at dequeue, `''` -> value, + * never changes), and `error_fingerprint` (empty until a terminal error status, then fixed). Those + * go in PREWHERE to filter (and, for tags, use the skip index) before FINAL and before materialising + * the wide columns, which is what bounds memory on these scans. Columns that change as a run runs + * stay in WHERE (post-FINAL): `status` (lifecycle) and `machine_preset` (escalates on OOM retry). + * The `(organization_id, project_id, environment_id)` primary-key prefix and the `created_at` range + * also stay in WHERE so they keep driving primary-key and partition pruning. + */ function applyRunFiltersToQueryBuilder( queryBuilder: ClickhouseQueryBuilder, options: FilterRunsOptions @@ -426,28 +442,14 @@ function applyRunFiltersToQueryBuilder( environmentId: options.environmentId, }); - if (options.tasks && options.tasks.length > 0) { - queryBuilder.where("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks }); - } - - if (options.versions && options.versions.length > 0) { - queryBuilder.where("task_version IN {versions: Array(String)}", { - versions: options.versions, - }); - } - if (options.statuses && options.statuses.length > 0) { queryBuilder.where("status IN {statuses: Array(String)}", { statuses: options.statuses }); } - if (options.tags && options.tags.length > 0) { - // Both hasAny and hasAll are served by the tags bloom_filter skip index. - const tagsFn = options.tagsMatch === "all" ? "hasAll" : "hasAny"; - queryBuilder.where(`${tagsFn}(tags, {tags: Array(String)})`, { tags: options.tags }); - } - - if (options.scheduleId) { - queryBuilder.where("schedule_id = {scheduleId: String}", { scheduleId: options.scheduleId }); + if (options.machines && options.machines.length > 0) { + queryBuilder.where("machine_preset IN {machines: Array(String)}", { + machines: options.machines, + }); } // Period is a number of milliseconds duration @@ -467,49 +469,63 @@ function applyRunFiltersToQueryBuilder( queryBuilder.where("created_at <= fromUnixTimestamp64Milli({to: Int64})", { to: options.to }); } + if (options.tasks && options.tasks.length > 0) { + queryBuilder.prewhere("task_identifier IN {tasks: Array(String)}", { tasks: options.tasks }); + } + + if (options.versions && options.versions.length > 0) { + queryBuilder.prewhere("task_version IN {versions: Array(String)}", { + versions: options.versions, + }); + } + + if (options.tags && options.tags.length > 0) { + // Both hasAny and hasAll are served by the tags bloom_filter skip index. + const tagsFn = options.tagsMatch === "all" ? "hasAll" : "hasAny"; + queryBuilder.prewhere(`${tagsFn}(tags, {tags: Array(String)})`, { tags: options.tags }); + } + + if (options.scheduleId) { + queryBuilder.prewhere("schedule_id = {scheduleId: String}", { + scheduleId: options.scheduleId, + }); + } + if (typeof options.isTest === "boolean") { - queryBuilder.where("is_test = {isTest: Boolean}", { isTest: options.isTest }); + queryBuilder.prewhere("is_test = {isTest: Boolean}", { isTest: options.isTest }); } if (options.rootOnly) { - queryBuilder.where("root_run_id = ''"); + queryBuilder.prewhere("root_run_id = ''"); } if (options.batchId) { - queryBuilder.where("batch_id = {batchId: String}", { batchId: options.batchId }); + queryBuilder.prewhere("batch_id = {batchId: String}", { batchId: options.batchId }); } if (options.bulkId) { - queryBuilder.where("hasAny(bulk_action_group_ids, {bulkActionGroupIds: Array(String)})", { + queryBuilder.prewhere("hasAny(bulk_action_group_ids, {bulkActionGroupIds: Array(String)})", { bulkActionGroupIds: [options.bulkId], }); } if (options.runId && options.runId.length > 0) { // it's important that in the query it's "runIds", otherwise it clashes with the cursor which is called "runId" - queryBuilder.where("friendly_id IN {runIds: Array(String)}", { + queryBuilder.prewhere("friendly_id IN {runIds: Array(String)}", { runIds: options.runId.map((runId) => RunId.toFriendlyId(runId)), }); } if (options.queues && options.queues.length > 0) { - queryBuilder.where("queue IN {queues: Array(String)}", { queues: options.queues }); + queryBuilder.prewhere("queue IN {queues: Array(String)}", { queues: options.queues }); } if (options.regions && options.regions.length > 0) { - queryBuilder.where("if(region != '', region, worker_queue) IN {regions: Array(String)}", { - regions: options.regions, - }); - } - - if (options.machines && options.machines.length > 0) { - queryBuilder.where("machine_preset IN {machines: Array(String)}", { - machines: options.machines, - }); + queryBuilder.prewhere("region IN {regions: Array(String)}", { regions: options.regions }); } if (options.errorId) { - queryBuilder.where("error_fingerprint = {errorFingerprint: String}", { + queryBuilder.prewhere("error_fingerprint = {errorFingerprint: String}", { errorFingerprint: ErrorId.toId(options.errorId), }); } @@ -520,11 +536,11 @@ function applyRunFiltersToQueryBuilder( const effectiveKinds = includesStandard ? [...options.taskKinds, ""] : options.taskKinds; if (effectiveKinds.length === 1) { - queryBuilder.where("task_kind = {taskKind: String}", { + queryBuilder.prewhere("task_kind = {taskKind: String}", { taskKind: effectiveKinds[0]!, }); } else { - queryBuilder.where("task_kind IN {taskKinds: Array(String)}", { + queryBuilder.prewhere("task_kind IN {taskKinds: Array(String)}", { taskKinds: effectiveKinds, }); } diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts index 066af4be101..6eff2f9336f 100644 --- a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts @@ -39,7 +39,7 @@ export async function getBillableEnvironmentsForBillingLimit( export async function createBillingLimitRunsRepository(organizationId: string) { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); return new RunsRepository({ @@ -95,7 +95,7 @@ export async function countBillableQueuedRunsForOrganization( ): Promise { const client = clickhouse ?? - (await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard")); + (await clickhouseFactory.getClickhouseForOrganization(organizationId, "runsList")); const queryBuilder = client.taskRuns.countQueryBuilder({ settings: { max_execution_time: BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S }, diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts index d3cd77b143c..b19f6e4bc69 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts @@ -115,7 +115,7 @@ export class BulkActionService extends BaseService { // Count the runs that will be affected by the bulk action const clickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, @@ -275,7 +275,7 @@ export class BulkActionService extends BaseService { const clickhouse = await clickhouseFactory.getClickhouseForOrganization( group.project.organizationId, - "standard" + "runsList" ); const runsRepository = new RunsRepository({ clickhouse, diff --git a/apps/webapp/test/runsListClickhouseSettings.test.ts b/apps/webapp/test/runsListClickhouseSettings.test.ts new file mode 100644 index 00000000000..fc066b60369 --- /dev/null +++ b/apps/webapp/test/runsListClickhouseSettings.test.ts @@ -0,0 +1,66 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { clickhouseTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { z } from "zod"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("runs-list ClickHouse protection settings", () => { + clickhouseTest( + "server-side max_execution_time kills a slow read, and readonly=2 does not block the caps", + async ({ clickhouseContainer }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-settings-test", + requestTimeoutMs: 30_000, + clickhouseSettings: { + max_execution_time: 1, + timeout_before_checking_execution_speed: 0, + max_threads: 2, + readonly: "2", + }, + }); + + const slow = clickhouse.reader.query({ + name: "slow-read", + query: "SELECT sum(number) AS total FROM numbers(1000000000000)", + schema: z.object({ total: z.number() }), + }); + const [slowError] = await slow({}); + + expect(slowError).not.toBeNull(); + expect(slowError?.message.toLowerCase()).toMatch(/timeout|exceeded/); + + const fast = clickhouse.reader.query({ + name: "fast-read", + query: "SELECT 1 AS one", + schema: z.object({ one: z.number() }), + }); + const [fastError, rows] = await fast({}); + + expect(fastError).toBeNull(); + expect(rows).toEqual([{ one: 1 }]); + } + ); + + clickhouseTest( + "readonly=2 rejects writes while permitting reads", + async ({ clickhouseContainer }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-readonly-test", + clickhouseSettings: { readonly: "2" }, + }); + + const write = clickhouse.reader.query({ + name: "write-under-readonly", + query: "CREATE TABLE trigger_dev.runs_list_readonly_probe (id UInt8) ENGINE = Memory", + schema: z.object({}), + }); + const [writeError] = await write({}); + + expect(writeError).not.toBeNull(); + expect(writeError?.message.toLowerCase()).toMatch(/readonly|read-only|read only/); + } + ); +}); diff --git a/apps/webapp/test/runsListQueryShape.test.ts b/apps/webapp/test/runsListQueryShape.test.ts new file mode 100644 index 00000000000..c0411b6fc7a --- /dev/null +++ b/apps/webapp/test/runsListQueryShape.test.ts @@ -0,0 +1,124 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +const DAY_MS = 24 * 60 * 60 * 1000; + +describe("runs list query shape (PREWHERE routing under FINAL)", () => { + containerTest( + "keeps status post-FINAL and returns old pending runs (no date clamp)", + async ({ clickhouseContainer, prisma }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "query-shape-test", + }); + + const ctx = await seedParents(prisma, "shape"); + + const completed = await createRun(prisma, ctx, { friendlyId: "run_completed" }); + const pendingRecent = await createRun(prisma, ctx, { friendlyId: "run_pending_recent" }); + const pendingOld = await createRun(prisma, ctx, { friendlyId: "run_pending_old" }); + + const base = { + taskIdentifier: "webhook.deliver", + runTags: ["booking:T"], + createdAt: new Date(Date.now() - 1 * DAY_MS), + }; + + await insertTaskRunV2Rows(clickhouse, [ + { ...completed, ...base, status: "PENDING", updatedAt: new Date(Date.now() - 2 * DAY_MS) }, + { + ...completed, + ...base, + status: "COMPLETED", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + { + ...pendingRecent, + ...base, + status: "PENDING", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + { + ...pendingOld, + ...base, + status: "PENDING", + createdAt: new Date(Date.now() - 60 * DAY_MS), + updatedAt: new Date(Date.now() - 60 * DAY_MS), + }, + ]); + + const repository = new RunsRepository({ prisma, clickhouse }); + + const { runIds } = await repository.listRunIds({ + page: { size: 10 }, + period: "365d", + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + tasks: ["webhook.deliver"], + tags: ["booking:T"], + statuses: ["PENDING", "DELAYED"], + }); + + expect(runIds.sort()).toEqual([pendingOld.id, pendingRecent.id].sort()); + } + ); + + containerTest( + "region filter matches a dequeued run by its region; a still-queued run is not matched", + async ({ clickhouseContainer, prisma }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "query-shape-region-test", + }); + + const ctx = await seedParents(prisma, "region"); + const dequeued = await createRun(prisma, ctx, { friendlyId: "run_dequeued" }); + const queued = await createRun(prisma, ctx, { friendlyId: "run_queued" }); + + const shared = { + taskIdentifier: "webhook.deliver", + createdAt: new Date(Date.now() - 1 * DAY_MS), + }; + + await insertTaskRunV2Rows(clickhouse, [ + { ...dequeued, ...shared, region: "", updatedAt: new Date(Date.now() - 2 * DAY_MS) }, + { + ...dequeued, + ...shared, + region: "us-east-1", + updatedAt: new Date(Date.now() - 1 * DAY_MS), + }, + ]); + await insertTaskRunV2Rows(clickhouse, [ + { ...queued, ...shared, region: "", updatedAt: new Date(Date.now() - 1 * DAY_MS) }, + ]); + + const repository = new RunsRepository({ prisma, clickhouse }); + const listArgs = { + page: { size: 10 } as const, + period: "365d", + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const byRegion = await repository.listRunIds({ ...listArgs, regions: ["us-east-1"] }); + expect(byRegion.runIds).toEqual([dequeued.id]); + + const otherRegion = await repository.listRunIds({ ...listArgs, regions: ["us-west-2"] }); + expect(otherRegion.runIds).toEqual([]); + } + ); +}); diff --git a/internal-packages/clickhouse/src/client/queryBuilder.ts b/internal-packages/clickhouse/src/client/queryBuilder.ts index bcdc68089c9..7b4f172e931 100644 --- a/internal-packages/clickhouse/src/client/queryBuilder.ts +++ b/internal-packages/clickhouse/src/client/queryBuilder.ts @@ -12,6 +12,7 @@ export type WhereCondition = { export class ClickhouseQueryBuilder { private name: string; private baseQuery: string; + private prewhereClauses: string[] = []; private whereClauses: string[] = []; private havingClauses: string[] = []; private params: QueryParams = {}; @@ -42,6 +43,28 @@ export class ClickhouseQueryBuilder { return this; } + /** + * Adds a PREWHERE clause. On a `... FINAL` base query, only use this for columns that are + * immutable or additive across a run's versions (e.g. task_identifier, tags): PREWHERE filters + * rows before FINAL reconciles versions, so a mutable column (e.g. status) would keep a stale + * version and drop the winning one. It filters before materialising the wide columns, which is + * what bounds memory on `task_runs_v2 FINAL` scans. + */ + prewhere(clause: string, params?: QueryParams): this { + this.prewhereClauses.push(clause); + if (params) { + Object.assign(this.params, params); + } + return this; + } + + prewhereIf(condition: any, clause: string, params?: QueryParams): this { + if (condition) { + this.prewhere(clause, params); + } + return this; + } + where(clause: string, params?: QueryParams): this { this.whereClauses.push(clause); if (params) { @@ -117,6 +140,9 @@ export class ClickhouseQueryBuilder { build(): { query: string; params: QueryParams } { let query = this.baseQuery; + if (this.prewhereClauses.length > 0) { + query += " PREWHERE " + this.prewhereClauses.join(" AND "); + } if (this.whereClauses.length > 0) { query += " WHERE " + this.whereClauses.join(" AND "); } From 036cf8d2c8d42d0a3fe0ef34be0475ba037c82f8 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski Date: Tue, 25 Aug 2026 13:30:54 +0200 Subject: [PATCH 10/28] chore(webapp): admin endpoint to backfill Vercel deployment external ids (#4770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skew protection resolves a run's worker by (environmentId, externalId, status=DEPLOYED). A miss parks the run and then expires it, so deployments predating the feature — which already carry the same value in commitSHA — need externalId populated to stay reachable. Vercel instant-rollback is the sharpest case, which is why the scope is the current promotion plus a recent window rather than current alone. Follows the existing backfill shape: admin PAT, keyset cursor over environments, per-environment action results, pMap, dryRun defaulting to true. Reuses normalizeExternalDeploymentId so a backfilled id is byte-identical to what a build writes, and the update re-checks externalId IS NULL so a deploy landing mid-backfill keeps its own id. Refs TRI-13464. --- ...min.api.v1.vercel-external-ids.backfill.ts | 74 ++++ .../vercelExternalIdBackfill.server.ts | 273 ++++++++++++ .../test/vercelExternalIdBackfill.test.ts | 419 ++++++++++++++++++ 3 files changed, 766 insertions(+) create mode 100644 apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts create mode 100644 apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts create mode 100644 apps/webapp/test/vercelExternalIdBackfill.test.ts diff --git a/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts new file mode 100644 index 00000000000..42de308be87 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts @@ -0,0 +1,74 @@ +import type { ActionFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +const BodySchema = z.object({ + cursor: z.string().optional(), + limit: z.number().int().min(1).max(500).default(50), + recentPerEnvironment: z.number().int().min(0).max(200).default(10), + parallelism: z.number().int().min(1).max(20).default(5), + dryRun: z.boolean().default(true), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method Not Allowed" }, { status: 405 }); + } + + const [bodyError, body] = await tryCatch(request.json()); + if (bodyError) { + return json({ error: bodyError.message }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(body); + if (!parsedBody.success) { + return json({ error: parsedBody.error.message }, { status: 400 }); + } + + const { cursor, limit, recentPerEnvironment, parallelism, dryRun } = parsedBody.data; + + logger.info("Vercel external id backfill starting", { + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }); + + const [error, result] = await tryCatch( + backfillVercelExternalIds({ + prisma, + replica: $replica, + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }) + ); + + if (error) { + logger.error("Vercel external id backfill failed", { cursor, error }); + return json({ error: error.message }, { status: 500 }); + } + + logger.info("Vercel external id backfill batch complete", { + dryRun, + cursor, + projectCount: result.projects, + environmentCount: result.environments.length, + summary: result.summary, + deployments: result.deployments, + next: result.next, + done: result.done, + }); + + return json(result); +} diff --git a/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts new file mode 100644 index 00000000000..cdbb17f0087 --- /dev/null +++ b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts @@ -0,0 +1,273 @@ +import type { PrismaClientOrTransaction } from "@trigger.dev/database"; +import { normalizeExternalDeploymentId, tryCatch } from "@trigger.dev/core/v3"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic"; +import pMap from "p-map"; +import { logger } from "~/services/logger.server"; + +type BackfillEnvironmentResult = { + /** An environment id, or a project id when `scope` is "project". */ + id: string; + /** Only set when the failure happened before any environment was resolved. */ + scope?: "project"; + action: "updated" | "would_update" | "skipped_nothing_eligible" | "error"; + eligible?: number; + written?: number; + error?: string; +}; + +export type BackfillResult = { + projects: number; + environments: BackfillEnvironmentResult[]; + summary: Record; + deployments: { eligible: number; written: number }; + next?: string; + done?: boolean; +}; + +export type BackfillOptions = { + prisma: PrismaClientOrTransaction; + replica: PrismaClientOrTransaction; + cursor?: string; + limit: number; + recentPerEnvironment: number; + parallelism: number; + dryRun: boolean; +}; + +type Candidate = { id: string; externalId: string }; + +/** + * Copy `commitSHA` into `externalId` for Vercel deployments that predate skew + * protection, one keyset page of connected projects at a time. + * + * Resolution reads (environmentId, externalId, status=DEPLOYED) and a miss parks + * the run rather than falling back, so a deployment that stores a commit SHA but + * no external id is unreachable to an app that sends one. + * + * `cursor` and `limit` are in OrganizationProjectIntegration ids, so a page is N + * connected projects and yields however many environments those hold. + */ +export async function backfillVercelExternalIds(options: BackfillOptions): Promise { + const { replica, cursor, limit, parallelism } = options; + + // Paginate over the connected projects rather than over environments. Driving + // from RuntimeEnvironment means "is this Vercel-connected" sits two joins away + // from the ordered column, so no index can serve filter and order together and + // every page has to build the whole matching set and sort it. Here the keyset + // runs on this table's primary key and the page is bounded by `take`. + const integrations = await replica.organizationProjectIntegration.findMany({ + where: { + deletedAt: null, + organizationIntegration: { service: "VERCEL", deletedAt: null }, + id: cursor ? { gt: cursor } : undefined, + }, + select: { id: true, projectId: true }, + orderBy: { id: "asc" }, + take: limit, + }); + + if (integrations.length === 0) { + return { + projects: 0, + environments: [], + summary: {}, + deployments: { eligible: 0, written: 0 }, + done: true, + }; + } + + const next = integrations[integrations.length - 1]?.id; + + // A project can hold more than one connection row, and reconnecting leaves the + // old one behind. Deduping keeps a page from walking the same environments twice. + const projectIds = [...new Set(integrations.map((integration) => integration.projectId))]; + + // One equality lookup per project rather than a single `projectId IN (...)`. A + // wide IN list tips the planner into seq-scanning RuntimeEnvironment, whereas an + // equality always rides projectId's index. These run concurrently anyway. + // Nothing in this mapper may throw. `stopOnError: false` does not isolate a + // rejected mapper: pMap still rejects the whole call with an AggregateError, + // which would cost the page its results and its `next` cursor. + const perProject = await pMap( + projectIds, + async (projectId): Promise => { + const [lookupError, environments] = await tryCatch( + replica.runtimeEnvironment.findMany({ + where: { projectId, type: { not: "DEVELOPMENT" } }, + select: { id: true }, + orderBy: { id: "asc" }, + }) + ); + + if (lookupError) { + logger.error("Vercel external id backfill could not list environments", { + projectId, + error: lookupError, + }); + return [{ id: projectId, scope: "project", action: "error", error: lookupError.message }]; + } + + const results: BackfillEnvironmentResult[] = []; + for (const environment of environments) { + results.push(await backfillEnvironment(environment.id, options)); + } + return results; + }, + { concurrency: parallelism, stopOnError: false } + ); + + const results = perProject.flat(); + + if (results.length === 0) { + return { + projects: projectIds.length, + environments: [], + summary: {}, + deployments: { eligible: 0, written: 0 }, + next, + }; + } + + const summary = results.reduce>((acc, result) => { + acc[result.action] = (acc[result.action] ?? 0) + 1; + return acc; + }, {}); + + const deployments = results.reduce( + (acc, result) => ({ + eligible: acc.eligible + (result.eligible ?? 0), + written: acc.written + (result.written ?? 0), + }), + { eligible: 0, written: 0 } + ); + + return { + projects: projectIds.length, + environments: results, + summary, + deployments, + next, + }; +} + +async function backfillEnvironment( + environmentId: string, + options: BackfillOptions +): Promise { + const [readError, candidates] = await tryCatch(findCandidates(environmentId, options)); + + if (readError) { + logger.error("Vercel external id backfill could not read deployments", { + environmentId, + error: readError, + }); + return { id: environmentId, action: "error", error: readError.message }; + } + + if (candidates.length === 0) { + return { id: environmentId, action: "skipped_nothing_eligible", eligible: 0 }; + } + + if (options.dryRun) { + return { id: environmentId, action: "would_update", eligible: candidates.length }; + } + + let written = 0; + + for (const candidate of candidates) { + const [writeError, result] = await tryCatch( + options.prisma.workerDeployment.updateMany({ + // Re-checking externalId lets a deploy landing mid-backfill keep the id it set. + where: { id: candidate.id, externalId: null }, + data: { externalId: candidate.externalId }, + }) + ); + + if (writeError) { + logger.error("Vercel external id backfill could not write a deployment", { + environmentId, + deploymentId: candidate.id, + error: writeError, + }); + return { + id: environmentId, + action: "error", + eligible: candidates.length, + written, + error: writeError.message, + }; + } + + written += result.count; + } + + return { id: environmentId, action: "updated", eligible: candidates.length, written }; +} + +/** + * The deployment holding the `current` promotion, plus the most recent DEPLOYED + * ones. Only DEPLOYED deployments are ever resolved, and `current` plus a recent + * window is what can still receive traffic. The window is there for Vercel + * instant-rollback, where the live app is an older commit than `current`. + */ +async function findCandidates( + environmentId: string, + { replica, recentPerEnvironment }: BackfillOptions +): Promise { + const select = { + id: true, + externalId: true, + commitSHA: true, + workerId: true, + status: true, + } as const; + + const [promotion, recent] = await Promise.all([ + replica.workerDeploymentPromotion.findFirst({ + where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL }, + select: { deployment: { select } }, + }), + recentPerEnvironment > 0 + ? replica.workerDeployment.findMany({ + where: { environmentId, status: "DEPLOYED" }, + select, + // id DESC, not createdAt: it matches [environmentId, status, id] exactly, so + // status stays in the index condition and the LIMIT bounds the scan. cuids sort + // by creation, and resolveExternalDeployment orders its candidates the same way. + orderBy: { id: "desc" }, + take: recentPerEnvironment, + }) + : Promise.resolve([]), + ]); + + const byId = new Map(); + for (const deployment of recent) { + byId.set(deployment.id, deployment); + } + if (promotion?.deployment) { + byId.set(promotion.deployment.id, promotion.deployment); + } + + const candidates: Candidate[] = []; + + for (const deployment of byId.values()) { + if ( + deployment.externalId !== null || + deployment.workerId === null || + deployment.status !== "DEPLOYED" + ) { + continue; + } + + // Reusing the live normalizer keeps a backfilled id byte-identical to what a + // build would have written. + const externalId = normalizeExternalDeploymentId(deployment.commitSHA ?? undefined); + if (!externalId) { + continue; + } + + candidates.push({ id: deployment.id, externalId }); + } + + return candidates; +} diff --git a/apps/webapp/test/vercelExternalIdBackfill.test.ts b/apps/webapp/test/vercelExternalIdBackfill.test.ts new file mode 100644 index 00000000000..ec087d6cbd4 --- /dev/null +++ b/apps/webapp/test/vercelExternalIdBackfill.test.ts @@ -0,0 +1,419 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +let seedCounter = 0; + +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); + +type SeedOptions = { + vercelConnected?: boolean; + environmentType?: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; +}; + +async function seedEnv(prisma: PrismaClient, slug: string, options: SeedOptions = {}) { + const { vercelConnected = true, environmentType = "PRODUCTION" } = options; + const n = seedCounter++; + + const organization = await prisma.organization.create({ + data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Proj ${slug}`, + slug: `proj-${slug}-${n}`, + organizationId: organization.id, + externalRef: `ext-${slug}-${n}`, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `env-${slug}-${n}`, + type: environmentType, + projectId: project.id, + organizationId: organization.id, + apiKey: `api-${slug}-${n}`, + pkApiKey: `pk-${slug}-${n}`, + shortcode: `sc-${slug}-${n}`, + }, + }); + + if (vercelConnected) { + const tokenReference = await prisma.secretReference.create({ + data: { key: `secret-${slug}-${n}` }, + }); + + const organizationIntegration = await prisma.organizationIntegration.create({ + data: { + friendlyId: `oi-${slug}-${n}`, + service: "VERCEL", + integrationData: {}, + tokenReferenceId: tokenReference.id, + organizationId: organization.id, + }, + }); + + await prisma.organizationProjectIntegration.create({ + data: { + organizationIntegrationId: organizationIntegration.id, + projectId: project.id, + externalEntityId: `vercel-project-${n}`, + integrationData: {}, + }, + }); + } + + return { organization, project, environment }; +} + +type SeedCtx = Awaited>; + +type DeploymentOptions = { + version: string; + commitSHA?: string | null; + externalId?: string | null; + status?: "DEPLOYED" | "FAILED" | "BUILDING"; + withWorker?: boolean; + createdAt?: Date; +}; + +async function seedDeployment(prisma: PrismaClient, ctx: SeedCtx, options: DeploymentOptions) { + const { + version, + commitSHA = SHA_A, + externalId = null, + status = "DEPLOYED", + withWorker = true, + createdAt, + } = options; + const n = seedCounter++; + + let workerId: string | undefined; + if (withWorker) { + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker-${n}`, + contentHash: `hash-${n}`, + projectId: ctx.project.id, + runtimeEnvironmentId: ctx.environment.id, + version, + metadata: {}, + }, + }); + workerId = worker.id; + } + + return prisma.workerDeployment.create({ + data: { + contentHash: `hash-${n}`, + friendlyId: `deployment-${n}`, + shortCode: `short-${n}`, + version, + status, + projectId: ctx.project.id, + environmentId: ctx.environment.id, + commitSHA, + externalId, + workerId, + ...(createdAt ? { createdAt } : {}), + }, + }); +} + +function run( + prisma: PrismaClient, + overrides: Partial[0]> = {} +) { + return backfillVercelExternalIds({ + prisma, + replica: prisma, + limit: 100, + recentPerEnvironment: 10, + parallelism: 5, + dryRun: false, + ...overrides, + }); +} + +describe("backfillVercelExternalIds", () => { + postgresTest("copies commitSHA into externalId for a Vercel deployment", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "copy"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(1); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_A); + }); + + postgresTest("a dry run reports the work and writes nothing", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dry"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma, { dryRun: true }); + + expect(result.summary.would_update).toBe(1); + expect(result.deployments.eligible).toBe(1); + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("never overwrites an existing externalId", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "keep"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + commitSHA: SHA_A, + externalId: SHA_B, + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_B); + }); + + postgresTest("skips projects with no Vercel integration", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "novercel", { vercelConnected: false }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.environments.find((e) => e.id === ctx.environment.id)).toBeUndefined(); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips development environments", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dev", { environmentType: "DEVELOPMENT" }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments that are not DEPLOYED", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "failed"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + status: "FAILED", + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments with no usable commitSHA", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "nosha"); + const missing = await seedDeployment(prisma, ctx, { version: "20260101.1", commitSHA: null }); + const blank = await seedDeployment(prisma, ctx, { version: "20260101.2", commitSHA: " " }); + const tooLong = await seedDeployment(prisma, ctx, { + version: "20260101.3", + commitSHA: "c".repeat(129), + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + for (const deployment of [missing, blank, tooLong]) { + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + } + }); + + postgresTest("skips deployments with no worker", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "noworker"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + withWorker: false, + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest( + "recentPerEnvironment bounds the window, and current is always included", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "window"); + + const oldest = await seedDeployment(prisma, ctx, { + version: "20260101.1", + createdAt: new Date("2026-01-01T00:00:00Z"), + }); + const newest = await seedDeployment(prisma, ctx, { + version: "20260101.2", + createdAt: new Date("2026-06-01T00:00:00Z"), + }); + + // Promote the oldest, so it can only be reached via the promotion arm. + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: oldest.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 1 }); + + expect(result.deployments.written).toBe(2); + + const afterOldest = await prisma.workerDeployment.findFirst({ where: { id: oldest.id } }); + const afterNewest = await prisma.workerDeployment.findFirst({ where: { id: newest.id } }); + expect(afterOldest?.externalId).toBe(SHA_A); + expect(afterNewest?.externalId).toBe(SHA_A); + } + ); + + postgresTest( + "recentPerEnvironment 0 backfills only the current promotion", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "currentonly"); + + const promoted = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + const other = await seedDeployment(prisma, ctx, { version: "20260101.2" }); + + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: promoted.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 0 }); + + expect(result.deployments.written).toBe(1); + + const afterPromoted = await prisma.workerDeployment.findFirst({ where: { id: promoted.id } }); + const afterOther = await prisma.workerDeployment.findFirst({ where: { id: other.id } }); + expect(afterPromoted?.externalId).toBe(SHA_A); + expect(afterOther?.externalId).toBeNull(); + } + ); + + postgresTest("is idempotent across a second run", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "idem"); + await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const first = await run(prisma); + const second = await run(prisma); + + expect(first.deployments.written).toBe(1); + expect(second.deployments.written).toBe(0); + }); + + postgresTest("paginates by connected project and reports done at the end", async ({ prisma }) => { + const first = await seedEnv(prisma, "page-a"); + const second = await seedEnv(prisma, "page-b"); + await seedDeployment(prisma, first, { version: "20260101.1" }); + await seedDeployment(prisma, second, { version: "20260101.1" }); + + const integrations = await prisma.organizationProjectIntegration.findMany({ + where: { projectId: { in: [first.project.id, second.project.id] } }, + select: { id: true, projectId: true }, + orderBy: { id: "asc" }, + }); + expect(integrations).toHaveLength(2); + + const page = await run(prisma, { limit: 1, dryRun: true }); + expect(page.projects).toBe(1); + expect(page.next).toBe(integrations[0]?.id); + + const firstPageProject = integrations[0]?.projectId; + const expectedEnvironment = + firstPageProject === first.project.id ? first.environment.id : second.environment.id; + expect(page.environments.map((e) => e.id)).toEqual([expectedEnvironment]); + + const secondPage = await run(prisma, { limit: 1, cursor: page.next, dryRun: true }); + expect(secondPage.projects).toBe(1); + expect(secondPage.environments.map((e) => e.id)).not.toEqual([expectedEnvironment]); + + const exhausted = await run(prisma, { cursor: secondPage.next, dryRun: true }); + expect(exhausted.done).toBe(true); + expect(exhausted.environments).toHaveLength(0); + }); + + postgresTest("a project with two connection rows is not walked twice", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dupe"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + // Reconnecting leaves the earlier row in place. + const existing = await prisma.organizationProjectIntegration.findFirst({ + where: { projectId: ctx.project.id }, + select: { organizationIntegrationId: true }, + }); + await prisma.organizationProjectIntegration.create({ + data: { + organizationIntegrationId: existing!.organizationIntegrationId, + projectId: ctx.project.id, + externalEntityId: "vercel-project-dupe", + integrationData: {}, + }, + }); + + const result = await run(prisma, { dryRun: true }); + + expect(result.projects).toBe(1); + expect(result.environments.filter((e) => e.id === ctx.environment.id)).toHaveLength(1); + expect(result.deployments.eligible).toBe(1); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("a failed environment lookup does not cost the page", async ({ prisma }) => { + const first = await seedEnv(prisma, "fail-a"); + const second = await seedEnv(prisma, "fail-b"); + await seedDeployment(prisma, first, { version: "20260101.1" }); + await seedDeployment(prisma, second, { version: "20260101.1" }); + + // pMap rejects the whole call when a mapper throws, even with + // stopOnError: false, so the lookup has to convert its own failures. + const replica = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "runtimeEnvironment") { + return { + findMany: async () => { + throw new Error("simulated replica failure"); + }, + }; + } + return Reflect.get(target, prop, receiver); + }, + }) as unknown as PrismaClient; + + const result = await backfillVercelExternalIds({ + prisma, + replica, + limit: 100, + recentPerEnvironment: 10, + parallelism: 5, + dryRun: false, + }); + + expect(result.projects).toBeGreaterThanOrEqual(2); + expect(result.next).toBeDefined(); + expect(result.summary.error).toBe(result.projects); + expect(result.environments.every((e) => e.scope === "project")).toBe(true); + expect(result.environments.every((e) => e.error === "simulated replica failure")).toBe(true); + expect(result.deployments.written).toBe(0); + }); +}); From 45eaaa7bd761f8d9018cb12e28ca88d483611834 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:01:24 +0100 Subject: [PATCH 11/28] feat(run-store,testcontainers): execution-snapshot read comparator and shared test utilities (#4772) ## Summary Adds the read comparator for the in-progress migration of the run execution-snapshot log from Postgres to Redis. The comparator samples a single read against both stores, normalizes the two results to one shape, and reports any per-field difference with a tagged metric. It never serves a read itself: the diff layer imports only types, so it cannot hold a store client, and a test enforces that by failing if any value import appears. Also adds a combined Postgres-and-Redis test fixture and two shared test utilities (a cluster-slot assertion and a generic fault-injection harness) that the parallel Redis-store work reuses. Everything here is inert. Nothing constructs the comparator, so merging changes no runtime behavior. It becomes active only when a later change turns on compare mode. ## Notes The divergence classes separate real differences (scalar, ordering, waitpoint id set, validity, missing on one side) from two expected classes that must not be driven to zero: a rotated idempotency key, and a Redis-only surplus at a since-cursor tie. The since comparison is direction sensitive: a Postgres-only entry at the cursor is always a lost write, never an expected tie. --- internal-packages/run-store/src/index.ts | 1 + .../run-store/src/redisSnapshotStore.test.ts | 34 +- .../src/snapshotComparator.isolation.test.ts | 57 ++++ .../run-store/src/snapshotComparator.test.ts | 254 +++++++++++++++ .../run-store/src/snapshotComparator.ts | 291 ++++++++++++++++++ .../testcontainers/src/clusterSlot.test.ts | 42 +++ .../testcontainers/src/clusterSlot.ts | 34 ++ .../testcontainers/src/faultInjection.test.ts | 58 ++++ .../testcontainers/src/faultInjection.ts | 46 +++ .../src/heteroRunOpsWithRedis.test.ts | 23 ++ internal-packages/testcontainers/src/index.ts | 45 ++- 11 files changed, 850 insertions(+), 35 deletions(-) create mode 100644 internal-packages/run-store/src/snapshotComparator.isolation.test.ts create mode 100644 internal-packages/run-store/src/snapshotComparator.test.ts create mode 100644 internal-packages/run-store/src/snapshotComparator.ts create mode 100644 internal-packages/testcontainers/src/clusterSlot.test.ts create mode 100644 internal-packages/testcontainers/src/clusterSlot.ts create mode 100644 internal-packages/testcontainers/src/faultInjection.test.ts create mode 100644 internal-packages/testcontainers/src/faultInjection.ts create mode 100644 internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 3717dc01527..8893975cf12 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,3 +3,4 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./snapshotComparator.js"; diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts index 0c7b4c0720a..a9db5a0790d 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts @@ -1,7 +1,7 @@ // Unit suite for the raw Redis execution-snapshot store. Redis-only: the store holds no Prisma // reference, so no Postgres container is needed. import { expect, describe, vi } from "vitest"; -import { redisTest } from "@internal/testcontainers"; +import { redisTest, slotOf } from "@internal/testcontainers"; import { createRedisClient } from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; import { @@ -1283,45 +1283,23 @@ describe("expectedCur compare-and-set", () => { ); }); -// CRC16/XMODEM over a key's hash tag, per Redis's cluster hashing rule. CLUSTER KEYSLOT is -// unavailable on this standalone container ("cluster support disabled"), so the slot is computed -// here instead. Verified against the `cluster-key-slot` package's output for our key shapes. -function crc16(str: string): number { - let crc = 0; - for (let i = 0; i < str.length; i++) { - crc ^= str.charCodeAt(i) << 8; - for (let j = 0; j < 8; j++) { - crc = crc & 0x8000 ? (crc << 1) ^ 0x1021 : crc << 1; - crc &= 0xffff; - } - } - return crc; -} - -function hashSlot(key: string): number { - const start = key.indexOf("{"); - const end = start === -1 ? -1 : key.indexOf("}", start + 1); - const tag = start !== -1 && end !== -1 && end > start + 1 ? key.slice(start + 1, end) : key; - return crc16(tag) % 16384; -} - describe("hash tag and keyPrefix", () => { it("every key for one run lands in one cluster slot", () => { // Keys come from snapshotKeys() plus the wp: suffix the Lua prelude derives the same way, // with a keyPrefix prepended by hand as ioredis would. A dropped hash tag would split the slots. - // Pin the helper itself before trusting it: the published XMODEM check value, and two known + // Pin the shared helper before trusting it: the published XMODEM check value, and two known // slots (one matching cluster-key-slot, one a different run's tag as a negative control -- // otherwise a constant-valued crc16 would satisfy slots.size === 1 for the wrong reason). - expect(crc16("123456789")).toBe(0x31c3); - expect(hashSlot("engine:snap:{run_1}:e")).toBe(8108); - expect(hashSlot("engine:snap:{run_2}:e")).toBe(12239); + expect(slotOf("123456789")).toBe(0x31c3); + expect(slotOf("engine:snap:{run_1}:e")).toBe(8108); + expect(slotOf("engine:snap:{run_2}:e")).toBe(12239); const k = snapshotKeys("run_1"); const base = k.e.slice(0, -2); const keys = [k.e, k.idx, k.cur, k.seq, `${base}:wp:1`, `${base}:wp:2`].map( (key) => `engine:${key}` ); - const slots = new Set(keys.map(hashSlot)); + const slots = new Set(keys.map(slotOf)); expect(slots.size).toBe(1); }); diff --git a/internal-packages/run-store/src/snapshotComparator.isolation.test.ts b/internal-packages/run-store/src/snapshotComparator.isolation.test.ts new file mode 100644 index 00000000000..ae71d227b80 --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.isolation.test.ts @@ -0,0 +1,57 @@ +// Proves the Frozen rule: the comparator's VALUE-import set is empty. Every import it has is +// `import type`, erased at runtime, so the compiled module pulls in no Redis or Prisma client and +// cannot read. Goes red the instant any value import is added — a client, the barrel, or a dynamic +// import(). The detector is pinned against redisSnapshotStore.ts (which value-imports a client) so +// this cannot pass as a tautology. +import { expect, it, describe } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; + +const here = dirname(fileURLToPath(import.meta.url)); + +// Returns the module specifiers a file imports FOR VALUE (i.e. that survive to runtime). `import type` +// declarations and named blocks whose specifiers are all inline `type` are erased and excluded. +function valueImports(sourcePath: string): string[] { + const raw = readFileSync(sourcePath, "utf8"); + const out: string[] = []; + + // Statements are scanned on RAW source, anchored to line start (`^\s*import`), so a `//` comment + // line never matches and no stripping can hide a real import. Only the mid-line dynamic `import(` + // check runs on comment-stripped source. The pin test below guarantees the scan catches a real import. + const stripped = raw.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, ""); + if (/(^|[^.\w])import\s*\(/.test(stripped)) out.push(""); + + const importRe = /^\s*import\b([\s\S]*?)\bfrom\s*["']([^"']+)["']/gm; + for (let m = importRe.exec(raw); m !== null; m = importRe.exec(raw)) { + const clause = m[1]; + const spec = m[2]; + if (/^\s*type\b/.test(clause)) continue; // `import type ... from` + const named = clause.match(/\{([\s\S]*?)\}/); + // Strip inline `type Foo` specifiers, including an `as Bar` alias, before checking whether any + // value specifier remains. + const inlineType = /\btype\s+[A-Za-z_$][\w$]*(?:\s+as\s+[A-Za-z_$][\w$]*)?/g; + if (named && !/(^|,)\s*[A-Za-z_$]/.test(named[1].replace(inlineType, ""))) { + continue; // every named specifier is an inline `type` — nothing left for value + } + out.push(spec); + } + + // Bare side-effect imports (`import "x"`) run the module. + const bareRe = /^\s*import\s*["']([^"']+)["']/gm; + for (let m = bareRe.exec(raw); m !== null; m = bareRe.exec(raw)) out.push(m[1]); + + return out; +} + +describe("comparator read-isolation", () => { + it("the detector flags a real value import (pin against the store)", () => { + // redisSnapshotStore.ts value-imports @internal/redis, so a working detector MUST see it. + const storeImports = valueImports(resolve(here, "redisSnapshotStore.ts")); + expect(storeImports).toContain("@internal/redis"); + }); + + it("the comparator has no value imports — it is import-type-only and cannot read", () => { + expect(valueImports(resolve(here, "snapshotComparator.ts"))).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/snapshotComparator.test.ts b/internal-packages/run-store/src/snapshotComparator.test.ts new file mode 100644 index 00000000000..ce61386c85a --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.test.ts @@ -0,0 +1,254 @@ +import { expect, it, describe } from "vitest"; +import { + diffLatest, + diffSince, + normalizeFromRedis, + normalizeFromPg, + SnapshotComparator, + type DivergenceClass, + type NormalizedSnapshot, +} from "./snapshotComparator.js"; +import type { SnapshotRead } from "./redisSnapshotStore.js"; + +function norm(over: Partial = {}): NormalizedSnapshot { + const base: NormalizedSnapshot = { + id: "s1", + engine: "V2", + executionStatus: "RUN_CREATED", + description: "d", + isValid: true, + error: null, + previousSnapshotId: null, + runId: "r1", + runStatus: "PENDING", + batchId: null, + attemptNumber: null, + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + checkpointId: null, + workerId: null, + runnerId: null, + createdAt: 1000, + updatedAt: 1000, + metadata: null, + completedWaitpointOrder: [], + waitpointIdSet: [], + }; + return { ...base, ...over }; +} + +describe("diffLatest", () => { + it("no divergence when the two sides match", () => { + expect(diffLatest(norm(), norm())).toEqual([]); + }); + + it("reports a scalar difference by field", () => { + expect(diffLatest(norm(), norm({ executionStatus: "EXECUTING" }))).toEqual([ + { field: "executionStatus", class: "scalar", pg: "RUN_CREATED", redis: "EXECUTING" }, + ]); + }); + + it("compares createdAt and updatedAt by strict equality", () => { + expect(diffLatest(norm(), norm({ createdAt: 1001 }))).toEqual([ + { field: "createdAt", class: "scalar", pg: 1000, redis: 1001 }, + ]); + }); + + it("classifies a validity mismatch", () => { + const d = diffLatest(norm({ isValid: true }), norm({ isValid: false, error: "boom" })); + expect(d.map((x) => x.field).sort()).toEqual(["error", "isValid"]); + expect(d.find((x) => x.field === "isValid")!.class).toBe("validity"); + }); + + it("classifies completedWaitpointOrder differences as order, repeats significant", () => { + expect( + diffLatest( + norm({ completedWaitpointOrder: ["a", "a", "b"] }), + norm({ completedWaitpointOrder: ["a", "b"] }) + ) + ).toEqual([ + { field: "completedWaitpointOrder", class: "order", pg: ["a", "a", "b"], redis: ["a", "b"] }, + ]); + }); + + it("classifies waitpoint id set differences, order-insensitive", () => { + expect( + diffLatest(norm({ waitpointIdSet: ["a", "b"] }), norm({ waitpointIdSet: ["a", "b"] })) + ).toEqual([]); + const d2 = diffLatest(norm({ waitpointIdSet: ["a", "b"] }), norm({ waitpointIdSet: ["a"] })); + expect(d2[0]).toMatchObject({ field: "waitpointIdSet", class: "waitpointIdSet" }); + }); + + it("does NOT emit a divergence for a rotated idempotency key — invisible at id-set granularity", () => { + expect(diffLatest(norm({ waitpointIdSet: ["w1"] }), norm({ waitpointIdSet: ["w1"] }))).toEqual( + [] + ); + }); + + it("missingInRedis when the row exists only in Postgres", () => { + expect(diffLatest(norm(), null)).toEqual([ + expect.objectContaining({ class: "missingInRedis" }), + ]); + }); + + it("missingInPg when the row exists only in Redis", () => { + expect(diffLatest(null, norm())).toEqual([expect.objectContaining({ class: "missingInPg" })]); + }); + + it("raises unknownField for a key on neither the compared nor excluded list", () => { + const d = diffLatest(norm(), { ...norm(), somethingNew: 1 } as NormalizedSnapshot); + expect(d).toEqual([expect.objectContaining({ field: "somethingNew", class: "unknownField" })]); + }); + + it("normalizeFromRedis carries an unrecognised entry field, so unknownField fires on real input", () => { + const read: SnapshotRead = { + id: "s1", + seq: 1, + isValid: true, + raw: "{}", + entry: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "d", + runId: "r1", + runStatus: "PENDING", + createdAt: "2026-08-24T00:00:00.000Z", + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + mysteryField: "surprise", + }, + }; + const redis = normalizeFromRedis(read); + expect(redis.mysteryField).toBe("surprise"); // not dropped by normalization + const d = diffLatest( + norm({ id: "s1", createdAt: redis.createdAt, updatedAt: redis.updatedAt }), + redis + ); + expect(d).toEqual([ + expect.objectContaining({ field: "mysteryField", class: "unknownField", redis: "surprise" }), + ]); + }); + + it("surfaces an inherited-name key and does not pollute the prototype", () => { + // JSON.parse produces OWN keys for `toString` and `__proto__` (unlike an object literal). + const entry = JSON.parse( + '{"engine":"V2","executionStatus":"RUN_CREATED","description":"d","runId":"r1",' + + '"runStatus":"PENDING","createdAt":"2026-08-24T00:00:00.000Z","environmentId":"env",' + + '"environmentType":"DEVELOPMENT","projectId":"p","organizationId":"o",' + + '"toString":"surprise","__proto__":{"polluted":true}}' + ) as Record; + const read: SnapshotRead = { id: "s1", seq: 1, isValid: true, raw: "{}", entry }; + const n = normalizeFromRedis(read) as Record; + + expect(Object.keys(n)).toContain("toString"); // carried as an own key despite the inherited name + expect(n["toString"]).toBe("surprise"); + expect(Object.getPrototypeOf(n)).toBe(Object.prototype); // __proto__ skipped, no pollution + expect("polluted" in {}).toBe(false); + + const d = diffLatest( + norm({ id: "s1", createdAt: n.createdAt as number, updatedAt: n.updatedAt as number }), + n as NormalizedSnapshot + ); + expect(d.some((x) => x.field === "toString" && x.class === "unknownField")).toBe(true); + }); + + it("normalizeFromPg's waitpointIdSet is index-bearing only, matching the Redis read surface", () => { + // A non-indexed completed waitpoint is in the relation but not in completedWaitpointOrder; Redis's + // distinctIds (dedupe of order) does not expose it, so the PG side must not either. + const row = { + id: "s1", + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + isValid: true, + error: null, + previousSnapshotId: null, + runId: "r1", + runStatus: "EXECUTING", + batchId: null, + attemptNumber: null, + environmentId: "env", + environmentType: "DEVELOPMENT", + projectId: "p", + organizationId: "o", + checkpointId: null, + workerId: null, + runnerId: null, + createdAt: new Date(1000), + updatedAt: new Date(1000), + metadata: null, + completedWaitpointOrder: ["w_indexed"], + completedWaitpoints: [{ id: "w_indexed" }, { id: "w_nonindexed" }], + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const n = normalizeFromPg(row as any); + expect(n.waitpointIdSet).toEqual(["w_indexed"]); + }); +}); + +describe("diffSince", () => { + const cursor = { id: "s1", createdAtMs: 1000 }; + + it("a Postgres-only entry at the cursor ms is a lost append (missingInRedis), never a tie", () => { + const pg = [norm({ id: "s2", createdAt: 1000, previousSnapshotId: "s1" })]; + expect(diffSince({ pg, redis: [], cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "missingInRedis" }), + ]); + }); + + it("a Redis-only chain-boundary surplus at the cursor ms is expected:redisSurplusAtCursorTie", () => { + const redis = [norm({ id: "s2", createdAt: 1000, previousSnapshotId: "s1" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "expected:redisSurplusAtCursorTie" }), + ]); + }); + + it("a Redis-only surplus that is NOT a chain boundary is a real missingInPg", () => { + const redis = [norm({ id: "s3", createdAt: 1000, previousSnapshotId: "s2" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s3", class: "missingInPg" }), + ]); + }); + + it("a Redis-only surplus above the cursor ms is a real missingInPg", () => { + const redis = [norm({ id: "s2", createdAt: 1500, previousSnapshotId: "s1" })]; + expect(diffSince({ pg: [], redis, cursor })).toEqual([ + expect.objectContaining({ field: "s2", class: "missingInPg" }), + ]); + }); +}); + +describe("SnapshotComparator", () => { + it("shouldSample honours the injected rng and percent", () => { + expect(new SnapshotComparator({ samplePercent: 10, rng: () => 0.05 }).shouldSample()).toBe( + true + ); + expect(new SnapshotComparator({ samplePercent: 10, rng: () => 0.5 }).shouldSample()).toBe( + false + ); + }); + + it("record emits one metric per divergence, tagged by class and op, and returns void", () => { + const seen: Array<{ op: string; cls: DivergenceClass }> = []; + const cmp = new SnapshotComparator({ + samplePercent: 100, + metrics: { + recordDivergence: (op, cls) => seen.push({ op, cls }), + recordSample: () => {}, + }, + }); + const ret = cmp.record("getLatest", [ + { field: "executionStatus", class: "scalar" }, + { field: "idempotencyKey", class: "expected:rotatedIdempotencyKey" }, + ]); + expect(ret).toBeUndefined(); + expect(seen).toEqual([ + { op: "getLatest", cls: "scalar" }, + { op: "getLatest", cls: "expected:rotatedIdempotencyKey" }, + ]); + }); +}); diff --git a/internal-packages/run-store/src/snapshotComparator.ts b/internal-packages/run-store/src/snapshotComparator.ts new file mode 100644 index 00000000000..d9f0407c030 --- /dev/null +++ b/internal-packages/run-store/src/snapshotComparator.ts @@ -0,0 +1,291 @@ +// Compare-mode read comparator: PURE diff layer. It NEVER serves a read — it takes results the caller +// already obtained and reports how the two stores disagree, by field, with a class. Type-only imports +// of client types, so this module holds no Redis or Prisma client (proven by the isolation test). +import type { Prisma } from "@trigger.dev/database"; +import type { SnapshotRead } from "./redisSnapshotStore.js"; + +export type DivergenceClass = + | "missingInRedis" + | "missingInPg" + | "scalar" + | "order" + | "waitpointIdSet" + | "validity" + | "unknownField" + // Reserved shared vocabulary for the payload-comparing layer a later ticket adds. This module + // compares id sets and order, not record payloads, so it never emits this — a rotated idempotency + // key does not change a waitpoint id. Kept in the union so the metric tag space stays stable. + | "expected:rotatedIdempotencyKey" + | "expected:redisSurplusAtCursorTie"; + +export type SnapshotDivergence = { + field: string; + class: DivergenceClass; + pg?: unknown; + redis?: unknown; +}; + +// The read operations the comparator samples. Bounded so the `op` metric attribute cannot become a +// high-cardinality label (a caller cannot pass a run id or other unbounded value). +export type SnapshotReadOp = "getLatest" | "getById" | "getSince" | "getSnapshotWaitpointIds"; + +export type NormalizedSnapshot = { + [k: string]: unknown; + id: string; + createdAt: number; // ms + updatedAt: number; // ms + completedWaitpointOrder: string[]; + waitpointIdSet: string[]; + previousSnapshotId?: string | null; +}; + +// The 22 compared entry columns. EXCLUDED_FIELDS names the columns deliberately not compared; any key +// on a normalized entry that is on neither list raises `unknownField`, so a new column fails loudly. +export const COMPARED_FIELDS = [ + "id", + "engine", + "executionStatus", + "description", + "isValid", + "error", + "previousSnapshotId", + "runId", + "runStatus", + "batchId", + "attemptNumber", + "environmentId", + "environmentType", + "projectId", + "organizationId", + "checkpointId", + "workerId", + "runnerId", + "createdAt", + "updatedAt", + "metadata", +] as const; + +// lastHeartbeatAt: Postgres-only, never written by the current engine. Waitpoint/checkpoint payloads: +// expanded by the run-engine resolver, out of this module's scope. +export const EXCLUDED_FIELDS = ["lastHeartbeatAt", "checkpoint", "completedWaitpoints"] as const; + +const SCALAR_FIELDS = COMPARED_FIELDS.filter((f) => f !== "metadata"); + +const KNOWN_KEYS = new Set([ + ...COMPARED_FIELDS, + ...EXCLUDED_FIELDS, + "completedWaitpointOrder", + "waitpointIdSet", +]); + +// Carry a source key normalization does not recognise onto the normalized object, so the unknownField +// check sees it instead of it being silently dropped (a false clean comparison). Skips the +// prototype-pollution keys. No own-property guard is needed: the normalizer only ever sets KNOWN_KEYS, +// so a non-known source key is never already present and cannot overwrite a normalized value. +const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]); +function carryUnknownKeys(target: NormalizedSnapshot, source: Record): void { + for (const k of Object.keys(source)) { + if (DANGEROUS_KEYS.has(k)) continue; + if (!KNOWN_KEYS.has(k)) target[k] = source[k]; + } +} + +function canonicalJson(v: unknown): string { + if (v === null || typeof v !== "object") return JSON.stringify(v) ?? "null"; + if (Array.isArray(v)) return `[${v.map(canonicalJson).join(",")}]`; + const obj = v as Record; + const keys = Object.keys(obj).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`; +} + +export function normalizeFromPg( + row: Prisma.TaskRunExecutionSnapshotGetPayload<{ include: { completedWaitpoints: true } }> +): NormalizedSnapshot { + const n: NormalizedSnapshot = { + id: row.id, + engine: row.engine, + executionStatus: row.executionStatus, + description: row.description, + isValid: row.isValid, + error: row.error ?? null, + previousSnapshotId: row.previousSnapshotId ?? null, + runId: row.runId, + runStatus: row.runStatus, + batchId: row.batchId ?? null, + attemptNumber: row.attemptNumber ?? null, + environmentId: row.environmentId, + environmentType: row.environmentType, + projectId: row.projectId, + organizationId: row.organizationId, + checkpointId: row.checkpointId ?? null, + workerId: row.workerId ?? null, + runnerId: row.runnerId ?? null, + createdAt: row.createdAt.getTime(), + updatedAt: row.updatedAt.getTime(), + metadata: row.metadata ?? null, + completedWaitpointOrder: [...(row.completedWaitpointOrder ?? [])], + // Index-bearing distinct set, from completedWaitpointOrder, to match the Redis read surface + // (distinctIds = dedupe of `order`). The full relation holds non-indexed ids Redis does not + // expose here (payload-layer, out of scope), so comparing it would fire a spurious divergence. + waitpointIdSet: [...new Set(row.completedWaitpointOrder ?? [])].sort(), + }; + carryUnknownKeys(n, row as unknown as Record); + return n; +} + +export function normalizeFromRedis(read: SnapshotRead): NormalizedSnapshot { + const e = read.entry as Record; + const createdAtMs = new Date(String(e.createdAt)).getTime(); + const order = read.completedWaitpointIds?.order ?? []; + const idSet = [...(read.completedWaitpointIds?.distinctIds ?? [])].sort(); + const n: NormalizedSnapshot = { + id: read.id, + engine: (e.engine ?? "V2") as string, + executionStatus: e.executionStatus as string, + description: e.description as string, + isValid: read.isValid, + error: (e.error ?? null) as string | null, + previousSnapshotId: (e.previousSnapshotId ?? null) as string | null, + runId: e.runId as string, + runStatus: e.runStatus as string, + batchId: (e.batchId ?? null) as string | null, + attemptNumber: (e.attemptNumber ?? null) as number | null, + environmentId: e.environmentId as string, + environmentType: e.environmentType as string, + projectId: e.projectId as string, + organizationId: e.organizationId as string, + checkpointId: (e.checkpointId ?? null) as string | null, + workerId: (e.workerId ?? null) as string | null, + runnerId: (e.runnerId ?? null) as string | null, + createdAt: createdAtMs, + updatedAt: createdAtMs, // write-once row: updatedAt equals createdAt + metadata: e.metadata ?? null, + completedWaitpointOrder: [...order], + waitpointIdSet: idSet, + }; + carryUnknownKeys(n, e); + return n; +} + +function sameArray(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((x, i) => x === b[i]); +} + +function fieldDivergences(pg: NormalizedSnapshot, redis: NormalizedSnapshot): SnapshotDivergence[] { + const out: SnapshotDivergence[] = []; + + for (const f of SCALAR_FIELDS) { + if (pg[f] !== redis[f]) { + out.push({ + field: f, + class: f === "isValid" ? "validity" : "scalar", + pg: pg[f], + redis: redis[f], + }); + } + } + if (canonicalJson(pg.metadata) !== canonicalJson(redis.metadata)) { + out.push({ field: "metadata", class: "scalar", pg: pg.metadata, redis: redis.metadata }); + } + if (!sameArray(pg.completedWaitpointOrder, redis.completedWaitpointOrder)) { + out.push({ + field: "completedWaitpointOrder", + class: "order", + pg: pg.completedWaitpointOrder, + redis: redis.completedWaitpointOrder, + }); + } + if (!sameArray(pg.waitpointIdSet, redis.waitpointIdSet)) { + out.push({ + field: "waitpointIdSet", + class: "waitpointIdSet", + pg: pg.waitpointIdSet, + redis: redis.waitpointIdSet, + }); + } + // Unknown keys on EITHER side, so a new field in either store fails loudly. + for (const k of new Set([...Object.keys(pg), ...Object.keys(redis)])) { + if (!KNOWN_KEYS.has(k)) { + out.push({ field: k, class: "unknownField", pg: pg[k], redis: redis[k] }); + } + } + return out; +} + +export function diffLatest( + pg: NormalizedSnapshot | null, + redis: NormalizedSnapshot | null +): SnapshotDivergence[] { + if (pg && !redis) return [{ field: pg.id, class: "missingInRedis", pg }]; + if (redis && !pg) return [{ field: redis.id, class: "missingInPg", redis }]; + if (!pg || !redis) return []; + return fieldDivergences(pg, redis); +} + +export type SnapshotComparatorMetrics = { + recordDivergence(op: SnapshotReadOp, cls: DivergenceClass): void; + recordSample(op: SnapshotReadOp): void; +}; + +// Samples reads and records divergence metrics. Holds no store and returns nothing from record(), so +// it structurally cannot serve a read. samplePercent is injected, never read from env.server. +export class SnapshotComparator { + readonly #samplePercent: number; + readonly #metrics?: SnapshotComparatorMetrics; + readonly #rng: () => number; + + constructor(opts: { + samplePercent: number; + metrics?: SnapshotComparatorMetrics; + rng?: () => number; + }) { + this.#samplePercent = opts.samplePercent; + this.#metrics = opts.metrics; + this.#rng = opts.rng ?? Math.random; + } + + shouldSample(): boolean { + return this.#rng() * 100 < this.#samplePercent; + } + + record(op: SnapshotReadOp, divergences: SnapshotDivergence[]): void { + this.#metrics?.recordSample(op); + for (const d of divergences) this.#metrics?.recordDivergence(op, d.class); + } +} + +export function diffSince(args: { + pg: NormalizedSnapshot[]; + redis: NormalizedSnapshot[]; + cursor: { id: string; createdAtMs: number }; +}): SnapshotDivergence[] { + const { pg, redis, cursor } = args; + const byId = (xs: NormalizedSnapshot[]) => new Map(xs.map((x) => [x.id, x])); + const pgMap = byId(pg); + const redisMap = byId(redis); + const out: SnapshotDivergence[] = []; + + // Present on both: field-diff. + for (const [id, p] of pgMap) { + const r = redisMap.get(id); + if (r) out.push(...fieldDivergences(p, r)); + } + // Postgres-only: ALWAYS a lost append. A same-ms tie can never surface here, because Postgres's own + // window drops the same-ms entry too. So there is no "expected tie" on this side. + for (const [id, p] of pgMap) { + if (!redisMap.has(id)) out.push({ field: id, class: "missingInRedis", pg: p }); + } + // Redis-only: expected ONLY when it is a chain boundary sitting exactly on the cursor ms (the + // id-cursor getSince path keeps a same-ms entry that Postgres's `> cursor` drops). Anything else is + // a real surplus. + for (const [id, r] of redisMap) { + if (pgMap.has(id)) continue; + const isTie = r.createdAt === cursor.createdAtMs && r.previousSnapshotId === cursor.id; + out.push({ + field: id, + class: isTie ? "expected:redisSurplusAtCursorTie" : "missingInPg", + redis: r, + }); + } + return out; +} diff --git a/internal-packages/testcontainers/src/clusterSlot.test.ts b/internal-packages/testcontainers/src/clusterSlot.test.ts new file mode 100644 index 00000000000..a7bec4313fa --- /dev/null +++ b/internal-packages/testcontainers/src/clusterSlot.test.ts @@ -0,0 +1,42 @@ +import { expect, it, describe } from "vitest"; +import { slotOf, expectOneSlot } from "./clusterSlot"; + +describe("slotOf", () => { + it("matches the published CRC16/XMODEM check value and known slots", () => { + expect(slotOf("123456789")).toBe(0x31c3); + expect(slotOf("engine:snap:{run_1}:e")).toBe(8108); + expect(slotOf("engine:snap:{run_2}:e")).toBe(12239); + }); + + it("groups keys that share a non-empty tag into one slot", () => { + expect(slotOf("a{tag}b")).toBe(slotOf("c{tag}d")); + }); + + it("hashes the whole key when the tag is empty (not the empty tag)", () => { + // If the empty `{}` were used as the tag, these would collide; hashing the whole key keeps them apart. + expect(slotOf("a{}b")).not.toBe(slotOf("c{}d")); + }); + + it("hashes the whole key when a brace is unclosed (malformed tag)", () => { + // `b` is not a tag here (no closing brace), so these must not share a slot the way `{b}` would. + expect(slotOf("a{b")).not.toBe(slotOf("x{b")); + }); + + it("hashes UTF-8 bytes, matching Redis for a non-ASCII tag", () => { + // Redis (cluster-key-slot) hashes the UTF-8 bytes of `é` to slot 10180. + expect(slotOf("{é}")).toBe(10180); + }); +}); + +describe("expectOneSlot", () => { + it("passes when every key shares one slot", () => { + expect(() => expectOneSlot(["snap:{r}:e", "snap:{r}:idx", "snap:{r}:cur"])).not.toThrow(); + }); + it("passes for zero or one key", () => { + expect(() => expectOneSlot([])).not.toThrow(); + expect(() => expectOneSlot(["snap:{r}:e"])).not.toThrow(); + }); + it("throws when two keys fall in different slots", () => { + expect(() => expectOneSlot(["snap:{run_1}:e", "snap:{run_2}:e"])).toThrow(/slot/i); + }); +}); diff --git a/internal-packages/testcontainers/src/clusterSlot.ts b/internal-packages/testcontainers/src/clusterSlot.ts new file mode 100644 index 00000000000..ccf74942176 --- /dev/null +++ b/internal-packages/testcontainers/src/clusterSlot.ts @@ -0,0 +1,34 @@ +// CRC16/XMODEM over a key's hash tag, computed here because CLUSTER KEYSLOT is unavailable on a +// standalone test container. Pinned against the cluster-key-slot package for our key shapes. Hashes +// UTF-8 BYTES (as Redis does), not UTF-16 code units, so a non-ASCII key still matches Redis's slot. + +function crc16(str: string): number { + let crc = 0; + for (const byte of Buffer.from(str, "utf8")) { + crc ^= byte << 8; + for (let j = 0; j < 8; j++) { + crc = crc & 0x8000 ? (crc << 1) ^ 0x1021 : crc << 1; + crc &= 0xffff; + } + } + return crc; +} + +/** The Redis cluster slot (0–16383) for a key, honouring `{…}` hash-tag extraction. */ +export function slotOf(key: string): number { + const start = key.indexOf("{"); + const end = start === -1 ? -1 : key.indexOf("}", start + 1); + const tag = start !== -1 && end !== -1 && end > start + 1 ? key.slice(start + 1, end) : key; + return crc16(tag) % 16384; +} + +/** Throws unless every key maps to one slot. A `[]` or single-key input passes. */ +export function expectOneSlot(keys: string[]): void { + if (keys.length <= 1) return; + const slots = new Set(keys.map(slotOf)); + if (slots.size !== 1) { + throw new Error( + `expected all keys in one cluster slot, got ${slots.size}: ${JSON.stringify(keys)}` + ); + } +} diff --git a/internal-packages/testcontainers/src/faultInjection.test.ts b/internal-packages/testcontainers/src/faultInjection.test.ts new file mode 100644 index 00000000000..fbcce7dc955 --- /dev/null +++ b/internal-packages/testcontainers/src/faultInjection.test.ts @@ -0,0 +1,58 @@ +import { expect, it, describe } from "vitest"; +import { createFaultInjector } from "./faultInjection"; + +type B = "afterPgBeforeRedis" | "midFlushRetry"; +class TestFault extends Error { + constructor(readonly boundary: B) { + super(`injected at ${boundary}`); + this.name = "TestFault"; + } +} +const make = () => createFaultInjector({ error: (b) => new TestFault(b) }); + +describe("createFaultInjector", () => { + it("does not throw when nothing is armed", () => { + const f = make(); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r1" })).not.toThrow(); + expect(f.fired("afterPgBeforeRedis")).toBe(0); + }); + + it("throws the injected error while armed, and counts each throw", () => { + const f = make(); + f.arm("afterPgBeforeRedis"); + expect(() => f.hook("afterPgBeforeRedis")).toThrow(TestFault); + expect(f.fired("afterPgBeforeRedis")).toBe(1); + }); + + it("times limits the number of throws", () => { + const f = make(); + f.arm("midFlushRetry", { times: 2 }); + expect(() => f.hook("midFlushRetry")).toThrow(); + expect(() => f.hook("midFlushRetry")).toThrow(); + expect(() => f.hook("midFlushRetry")).not.toThrow(); + expect(f.fired("midFlushRetry")).toBe(2); + }); + + it("runId scopes throws to the matching run only", () => { + const f = make(); + f.arm("afterPgBeforeRedis", { runId: "r1" }); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r2" })).not.toThrow(); + expect(() => f.hook("afterPgBeforeRedis", { runId: "r1" })).toThrow(); + expect(f.fired("afterPgBeforeRedis")).toBe(1); + }); + + it("rejects a non-integer or negative times, but allows the default (Infinity)", () => { + const f = make(); + expect(() => f.arm("midFlushRetry", { times: Number.NaN })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry", { times: 1.5 })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry", { times: -1 })).toThrow(RangeError); + expect(() => f.arm("midFlushRetry")).not.toThrow(); // unlimited + }); + + it("disarm clears a boundary", () => { + const f = make(); + f.arm("afterPgBeforeRedis"); + f.disarm("afterPgBeforeRedis"); + expect(() => f.hook("afterPgBeforeRedis")).not.toThrow(); + }); +}); diff --git a/internal-packages/testcontainers/src/faultInjection.ts b/internal-packages/testcontainers/src/faultInjection.ts new file mode 100644 index 00000000000..826cbbb3c36 --- /dev/null +++ b/internal-packages/testcontainers/src/faultInjection.ts @@ -0,0 +1,46 @@ +// Test-only fault-injection harness, shared by the snapshot decorator (crash-gap) and the waitpoint +// lane. Generic over the boundary union; the caller passes the error constructor, so this package +// takes no dependency on @internal/run-store (which would close a dependency cycle). The armed hook +// is SYNCHRONOUS: a crash at a write boundary must interrupt before the next write. + +export type FaultInjector = { + arm(boundary: TBoundary, opts?: { times?: number; runId?: string }): void; + disarm(boundary?: TBoundary): void; + hook: (boundary: TBoundary, context?: { runId?: string }) => void; + fired(boundary: TBoundary): number; +}; + +type Armed = { remaining: number; runId?: string }; + +export function createFaultInjector(opts: { + error: (boundary: TBoundary) => Error; +}): FaultInjector { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary, o) { + const times = o?.times ?? Infinity; + if (times !== Infinity && (!Number.isInteger(times) || times < 0)) { + throw new RangeError("times must be a non-negative integer or Infinity"); + } + armed.set(boundary, { remaining: times, runId: o?.runId }); + }, + disarm(boundary) { + if (boundary === undefined) armed.clear(); + else armed.delete(boundary); + }, + hook: (boundary, context) => { + const a = armed.get(boundary); + if (!a || a.remaining <= 0) return; + if (a.runId !== undefined && a.runId !== context?.runId) return; + a.remaining -= 1; + if (a.remaining <= 0) armed.delete(boundary); + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + throw opts.error(boundary); + }, + fired(boundary) { + return counts.get(boundary) ?? 0; + }, + }; +} diff --git a/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts b/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts new file mode 100644 index 00000000000..192fd877a17 --- /dev/null +++ b/internal-packages/testcontainers/src/heteroRunOpsWithRedis.test.ts @@ -0,0 +1,23 @@ +import { expect } from "vitest"; +import Redis from "ioredis"; +import { heteroRunOpsWithRedisTest } from "./index"; + +heteroRunOpsWithRedisTest( + "provides both Postgres clients and a live Redis", + async ({ prisma14, prisma17, redisOptions }) => { + const a = await prisma14.$queryRaw`SELECT 1 as ok`; + const b = await prisma17.$queryRaw`SELECT 1 as ok`; + expect(a).toEqual([{ ok: 1 }]); + expect(b).toEqual([{ ok: 1 }]); + + const redis = new Redis(redisOptions); + try { + expect(await redis.dbsize()).toBe(0); + await redis.set("k", "v"); + expect(await redis.get("k")).toBe("v"); + } finally { + await redis.quit(); + } + }, + 120_000 +); diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 820ca827aec..8cdaee8571b 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -431,14 +431,19 @@ type HeteroRunOpsPostgresTestContext = { // control-plane schema on PG14 (legacy), prisma17 is a RunOpsPrismaClient over the dedicated SUBSET // schema on a SEPARATE PG17 container. Lets a test prove the two sides carry different schemas // without disturbing the existing heteroPostgresTest (which keeps the full schema on both sides). -export const heteroRunOpsPostgresTest = test.extend({ - postgresContainer14: async ({}, use) => { +// The six hetero run-ops fixtures, shared by heteroRunOpsPostgresTest and heteroRunOpsWithRedisTest +// so the two cannot drift. +const heteroRunOpsFixtures = { + postgresContainer14: async ({}, use: Use) => { await use(await getWorkerPostgresContainer()); }, - postgresContainer17: async ({}, use) => { + postgresContainer17: async ({}, use: Use) => { await use(await getRunOpsWorkerPostgresContainer17()); }, - uri14: async ({ postgresContainer14 }, use) => { + uri14: async ( + { postgresContainer14 }: { postgresContainer14: StartedPostgreSqlContainer }, + use: Use + ) => { const baseUri = postgresContainer14.getConnectionUri(); const cloneDb = `heteroRunOps14_${pgCloneCounter++}`; await createDatabaseFromTemplate(baseUri, cloneDb); @@ -448,7 +453,10 @@ export const heteroRunOpsPostgresTest = test.extend { + uri17: async ( + { postgresContainer17 }: { postgresContainer17: StartedPostgreSqlContainer }, + use: Use + ) => { const baseUri = postgresContainer17.getConnectionUri(); const cloneDb = `heteroRunOps17_${pgCloneCounter++}`; await createDatabaseFromTemplate(baseUri, cloneDb); @@ -458,7 +466,7 @@ export const heteroRunOpsPostgresTest = test.extend { + prisma14: async ({ uri14 }: { uri14: string }, use: Use) => { const prisma = new PrismaClient({ datasources: { db: { url: uri14 } } }); try { await use(prisma); @@ -466,7 +474,7 @@ export const heteroRunOpsPostgresTest = test.extend { + prisma17: async ({ uri17 }: { uri17: string }, use: Use) => { const prisma = new RunOpsPrismaClient({ datasources: { db: { url: uri17 } } }); try { await use(prisma); @@ -474,6 +482,10 @@ export const heteroRunOpsPostgresTest = test.extend({ + ...heteroRunOpsFixtures, }); type ThreeDbRunOpsPostgresTestContext = { @@ -635,6 +647,22 @@ const flushRedis = async ( await use(); }; +type HeteroRunOpsWithRedisContext = HeteroRunOpsPostgresTestContext & { + redisContainer: StartedRedisContainer; + resetRedis: void; + redisOptions: RedisOptions; +}; + +// heteroRunOpsPostgresTest (PG14 + PG17, dedicated-schema run-ops) composed with the WORKER-SCOPED +// Redis container — boots once per worker, FLUSHALL between tests, matching containerTest. Not +// postgresAndRedisTest, which boots a container per test and times out under load. +export const heteroRunOpsWithRedisTest = test.extend({ + ...heteroRunOpsFixtures, + redisContainer: [bootWorkerRedis, { scope: "worker" }], + resetRedis: [flushRedis, { auto: true }], + redisOptions, +}); + type RedisTestContext = { redisContainer: StartedRedisContainer; resetRedis: void; @@ -980,3 +1008,6 @@ export const postgresAndMinioTest = withWarmup( await getWorkerPostgresContainer(); } ); + +export { slotOf, expectOneSlot } from "./clusterSlot"; +export { createFaultInjector, type FaultInjector } from "./faultInjection"; From 1eda438a413c5e59844045f7849305d7ef45cc1e Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 15:37:43 +0200 Subject: [PATCH 12/28] feat(webapp): put the admin dashboard behind an env var flag (#4774) Adds an `ADMIN_DASHBOARD_ENABLED` env var (default: enabled) that turns the admin dashboard and user impersonation off for an entire instance. When disabled: - every admin dashboard page redirects away, and the admin navigation isn't rendered - existing impersonation cookies are ignored, and any lingering session is actively terminated with an audit record - every flow that could start an impersonation responds 404, and no impersonation tokens are minted Stopping an impersonation always works regardless of the flag, so nothing gets stuck. Machine-to-machine admin API endpoints are not affected. The variable is documented for self-hosters; instances that don't set it are unaffected. --- .../admin-dashboard-enabled-flag.md | 6 ++ apps/webapp/app/env.server.ts | 2 + apps/webapp/app/hooks/useUser.ts | 6 ++ apps/webapp/app/models/admin.server.ts | 16 +++- apps/webapp/app/root.tsx | 11 ++- apps/webapp/app/routes/@.runs.$runParam.ts | 3 + .../_app.@.orgs.$organizationSlug.$.tsx | 5 + apps/webapp/app/routes/admin._index.tsx | 42 +++++---- apps/webapp/app/routes/admin.data-stores.tsx | 5 +- apps/webapp/app/routes/admin.impersonate.tsx | 6 +- apps/webapp/app/routes/admin.orgs.tsx | 30 +++--- .../app/routes/api.v1.plain.customer-cards.ts | 2 +- .../app/services/impersonation.server.ts | 16 ++++ .../routeBuilders/dashboardBuilder.server.ts | 3 +- .../routeBuilders/permissions.server.ts | 5 +- .../webapp/test/impersonationDisabled.test.ts | 92 +++++++++++++++++++ docs/self-hosting/env/webapp.mdx | 1 + 17 files changed, 210 insertions(+), 41 deletions(-) create mode 100644 .server-changes/admin-dashboard-enabled-flag.md create mode 100644 apps/webapp/test/impersonationDisabled.test.ts diff --git a/.server-changes/admin-dashboard-enabled-flag.md b/.server-changes/admin-dashboard-enabled-flag.md new file mode 100644 index 00000000000..7fe33ef3ec2 --- /dev/null +++ b/.server-changes/admin-dashboard-enabled-flag.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Self-hosted instances can now disable the admin dashboard and user impersonation entirely. See the self-hosting docs for the new setting. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c63c79d41d6..aec9cd82927 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -332,6 +332,8 @@ const EnvironmentSchema = z .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") .optional(), ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), + // Instance-level kill switch for the admin dashboard and user impersonation. + ADMIN_DASHBOARD_ENABLED: BoolEnv.default(true), REMIX_APP_PORT: z.string().optional(), // Opt-in, dev-only: stream this process's logs over a local telnet/TCP socket on this port. // Read directly from process.env in server.ts (before this schema loads); declared here for discoverability. diff --git a/apps/webapp/app/hooks/useUser.ts b/apps/webapp/app/hooks/useUser.ts index 2eed91b9734..08aff433192 100644 --- a/apps/webapp/app/hooks/useUser.ts +++ b/apps/webapp/app/hooks/useUser.ts @@ -48,6 +48,12 @@ export function useHasAdminAccess(matches?: UIMatch[]): boolean { const user = useOptionalUser(matches); const isImpersonating = useIsImpersonating(matches); const isViewingAsUser = useIsViewingAsUser(matches); + const routeMatch = useTypedMatchesData({ + id: "root", + matches, + }); + + if (routeMatch?.adminDashboardEnabled === false) return false; return (Boolean(user?.admin) || isImpersonating) && !isViewingAsUser; } diff --git a/apps/webapp/app/models/admin.server.ts b/apps/webapp/app/models/admin.server.ts index e93844dbaed..513b79c2261 100644 --- a/apps/webapp/app/models/admin.server.ts +++ b/apps/webapp/app/models/admin.server.ts @@ -5,16 +5,25 @@ import type { SearchParams } from "~/routes/admin._index"; import { clearImpersonationId, commitImpersonationSession, - getImpersonationId, + getRawImpersonationId, setImpersonationId, } from "~/services/impersonation.server"; import { authenticator } from "~/services/auth.server"; import { requireUser } from "~/services/session.server"; import { extractClientIp } from "~/utils/extractClientIp.server"; import { impersonationDestinationPath } from "~/utils/pathBuilder"; +import { env } from "~/env.server"; const pageSize = 20; +// 404, not 403, so a disabled instance doesn't advertise the feature. +// Stopping an impersonation is deliberately never gated. +export function requireAdminDashboardEnabled(): void { + if (!env.ADMIN_DASHBOARD_ENABLED) { + throw new Response("Not Found", { status: 404 }); + } +} + export async function adminGetUsers(userId: string, { page, search }: SearchParams) { page = page || 1; @@ -217,6 +226,8 @@ export async function redirectWithImpersonation( currentUser?: { id: string; admin: boolean }, prismaClient: PrismaClientOrTransaction = prisma ) { + requireAdminDashboardEnabled(); + const user = currentUser ?? (await requireUser(request)); if (!user.admin) { throw new Error("Unauthorized"); @@ -332,7 +343,8 @@ export async function startImpersonation( export async function clearImpersonation(request: Request, path: string) { const authUser = await authenticator.isAuthenticated(request); - const targetId = await getImpersonationId(request); + // Raw read: stops must audit and clear even with ADMIN_DASHBOARD_ENABLED off. + const targetId = await getRawImpersonationId(request); if (targetId && authUser?.userId) { const xff = request.headers.get("x-forwarded-for"); diff --git a/apps/webapp/app/root.tsx b/apps/webapp/app/root.tsx index 3cb547db4c7..de230f16a67 100644 --- a/apps/webapp/app/root.tsx +++ b/apps/webapp/app/root.tsx @@ -21,7 +21,8 @@ import { env } from "./env.server"; import { featuresForRequest } from "./features.server"; import { usePostHog } from "./hooks/usePostHog"; import { resolveThemePreference, useSystemThemeSync } from "./hooks/useSystemThemeSync"; -import { getImpersonationState } from "./services/impersonation.server"; +import { clearImpersonation } from "./models/admin.server"; +import { getImpersonationState, getRawImpersonationId } from "./services/impersonation.server"; import { getUser } from "./services/session.server"; import { normalizeIconContrast, @@ -117,6 +118,13 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { // the `user.isViewingAsUser` the server computes could disagree, and the // client-side admin UI would hide itself on a session that is not // impersonating. + // Flag off: terminate lingering impersonation sessions (audit + clear) + // rather than leaving a cookie that would resurrect on a later re-enable. + if (!env.ADMIN_DASHBOARD_ENABLED && (await getRawImpersonationId(request))) { + const url = new URL(request.url); + throw await clearImpersonation(request, `${url.pathname}${url.search}`); + } + const { isViewingAsUser } = await getImpersonationState(request, user?.id); const headers = new Headers(); @@ -126,6 +134,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { { user, isViewingAsUser, + adminDashboardEnabled: env.ADMIN_DASHBOARD_ENABLED, toastMessage, posthogProjectKey, posthogUiHost, diff --git a/apps/webapp/app/routes/@.runs.$runParam.ts b/apps/webapp/app/routes/@.runs.$runParam.ts index ed5ca156f38..d8ff7fd49d1 100644 --- a/apps/webapp/app/routes/@.runs.$runParam.ts +++ b/apps/webapp/app/routes/@.runs.$runParam.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { requireAdminDashboardEnabled } from "~/models/admin.server"; import { redirectWithErrorMessage } from "~/models/message.server"; import { requireUser } from "~/services/session.server"; import { impersonate, rootPath, v3RunPath, v3RunSpanPath } from "~/utils/pathBuilder"; @@ -13,6 +14,8 @@ const ParamsSchema = z.object({ }); export async function loader({ params, request }: LoaderFunctionArgs) { + requireAdminDashboardEnabled(); + const user = await requireUser(request); const { runParam } = ParamsSchema.parse(params); diff --git a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx index 2923a6fdeeb..0fc4f41d1a3 100644 --- a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx +++ b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx @@ -10,6 +10,7 @@ import { env } from "~/env.server"; import { clearImpersonation, findImpersonationTarget, + requireAdminDashboardEnabled, startImpersonation, } from "~/models/admin.server"; import { logger } from "~/services/logger.server"; @@ -26,6 +27,8 @@ import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; // here would drag server-only modules into the client build. export async function loader({ request, params }: LoaderFunctionArgs) { + requireAdminDashboardEnabled(); + const user = await requireUser(request); // If already impersonating, we need to clear the impersonation. Redirects are @@ -101,6 +104,8 @@ function refererOrigin(request: Request): string | undefined { } export async function action({ request, params }: ActionFunctionArgs) { + requireAdminDashboardEnabled(); + if (request.method.toLowerCase() !== "post") { return new Response("Method not allowed", { status: 405 }); } diff --git a/apps/webapp/app/routes/admin._index.tsx b/apps/webapp/app/routes/admin._index.tsx index 3005934d226..3741f499d38 100644 --- a/apps/webapp/app/routes/admin._index.tsx +++ b/apps/webapp/app/routes/admin._index.tsx @@ -2,6 +2,7 @@ import { MagnifyingGlassIcon } from "@heroicons/react/20/solid"; import { Form } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; +import { env } from "~/env.server"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { CopyableText } from "~/components/primitives/CopyableText"; import { Input } from "~/components/primitives/Input"; @@ -36,7 +37,7 @@ export const loader = dashboardLoader( } const result = await adminGetUsers(user.id, searchParams.params.getAll()); - return typedjson(result); + return typedjson({ ...result, impersonationEnabled: env.ADMIN_DASHBOARD_ENABLED }); } ); @@ -57,7 +58,8 @@ export const action = dashboardAction( ); export default function AdminDashboardRoute() { - const { users, filters, page, pageCount } = useTypedLoaderData(); + const { users, filters, page, pageCount, impersonationEnabled } = + useTypedLoaderData(); return (
{user.admin ? "✅" : ""} -
- - -
+ {impersonationEnabled && ( +
+ + +
+ )} ); diff --git a/apps/webapp/app/routes/admin.data-stores.tsx b/apps/webapp/app/routes/admin.data-stores.tsx index af4a15dfcae..acbb4675c42 100644 --- a/apps/webapp/app/routes/admin.data-stores.tsx +++ b/apps/webapp/app/routes/admin.data-stores.tsx @@ -25,6 +25,7 @@ import { TableRow, } from "~/components/primitives/Table"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { requireUser } from "~/services/session.server"; import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server"; import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; @@ -36,7 +37,7 @@ import { tryCatch } from "@trigger.dev/core/utils"; export const loader = async ({ request }: LoaderFunctionArgs) => { const user = await requireUser(request); - if (!user.admin) throw redirect("/"); + if (!user.admin || !env.ADMIN_DASHBOARD_ENABLED) throw redirect("/"); const dataStores = await prisma.organizationDataStore.findMany({ orderBy: { createdAt: "desc" }, @@ -72,7 +73,7 @@ const FormSchema = z.discriminatedUnion("_action", [AddSchema, UpdateSchema, Del export async function action({ request }: ActionFunctionArgs) { const user = await requireUser(request); - if (!user.admin) throw redirect("/"); + if (!user.admin || !env.ADMIN_DASHBOARD_ENABLED) throw redirect("/"); const formData = await request.formData(); diff --git a/apps/webapp/app/routes/admin.impersonate.tsx b/apps/webapp/app/routes/admin.impersonate.tsx index 458ed5b2a7e..46f711b26e8 100644 --- a/apps/webapp/app/routes/admin.impersonate.tsx +++ b/apps/webapp/app/routes/admin.impersonate.tsx @@ -4,7 +4,7 @@ import { type LoaderFunctionArgs, } from "@remix-run/server-runtime"; import { z } from "zod"; -import { redirectWithImpersonation } from "~/models/admin.server"; +import { redirectWithImpersonation, requireAdminDashboardEnabled } from "~/models/admin.server"; import { requireUser } from "~/services/session.server"; import { validateAndConsumeImpersonationToken } from "~/services/impersonation.server"; import { logger } from "~/services/logger.server"; @@ -20,6 +20,8 @@ async function handleImpersonationRequest(request: Request, userId: string): Pro } export const loader = async ({ request }: LoaderFunctionArgs) => { + requireAdminDashboardEnabled(); + const url = new URL(request.url); const impersonateUserId = url.searchParams.get("impersonate"); const impersonationToken = url.searchParams.get("impersonationToken"); @@ -50,6 +52,8 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { }; export async function action({ request }: ActionFunctionArgs) { + requireAdminDashboardEnabled(); + if (request.method.toLowerCase() !== "post") { return new Response("Method not allowed", { status: 405 }); } diff --git a/apps/webapp/app/routes/admin.orgs.tsx b/apps/webapp/app/routes/admin.orgs.tsx index 51cd9552325..132ad860bfd 100644 --- a/apps/webapp/app/routes/admin.orgs.tsx +++ b/apps/webapp/app/routes/admin.orgs.tsx @@ -3,6 +3,7 @@ import { Form } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { useState } from "react"; import { z } from "zod"; +import { env } from "~/env.server"; import { FeatureFlagsDialog } from "~/components/admin/FeatureFlagsDialog"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { CopyableText } from "~/components/primitives/CopyableText"; @@ -38,12 +39,13 @@ export const loader = dashboardLoader( } const result = await adminGetOrganizations(user.id, searchParams.params.getAll()); - return typedjson(result); + return typedjson({ ...result, impersonationEnabled: env.ADMIN_DASHBOARD_ENABLED }); } ); export default function AdminDashboardRoute() { - const { organizations, filters, page, pageCount } = useTypedLoaderData(); + const { organizations, filters, page, pageCount, impersonationEnabled } = + useTypedLoaderData(); const [flagsOrgId, setFlagsOrgId] = useState(null); const [flagsOpen, setFlagsOpen] = useState(false); @@ -127,17 +129,19 @@ export default function AdminDashboardRoute() { - - Impersonate - + {impersonationEnabled && ( + + Impersonate + + )} diff --git a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts index bfb9988bee2..de7f96be6c6 100644 --- a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts +++ b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts @@ -156,7 +156,7 @@ export async function action({ request }: ActionFunctionArgs) { * Derived from which lookup actually matched, not from whether an external id was *sent* — an * id that misses and falls through to email must not unlock impersonation. */ - const canImpersonate = Boolean(byExternalId); + const canImpersonate = Boolean(byExternalId) && env.ADMIN_DASHBOARD_ENABLED; // No matching user: still answer every requested key, with no data so Plain hides the cards. if (!user) { diff --git a/apps/webapp/app/services/impersonation.server.ts b/apps/webapp/app/services/impersonation.server.ts index aa850ba0468..bce88bfbc66 100644 --- a/apps/webapp/app/services/impersonation.server.ts +++ b/apps/webapp/app/services/impersonation.server.ts @@ -37,6 +37,14 @@ export function commitImpersonationSession(session: Session) { } export async function getImpersonationId(request: Request) { + if (!env.ADMIN_DASHBOARD_ENABLED) return undefined; + + return getRawImpersonationId(request); +} + +// Ignores ADMIN_DASHBOARD_ENABLED — only for terminating or auditing a session +// the gated reader no longer resolves, never for authorizing anything. +export async function getRawImpersonationId(request: Request) { const session = await getImpersonationSession(request); return session.get(IMPERSONATED_USER_ID_KEY) as string | undefined; @@ -74,6 +82,14 @@ export async function getImpersonationState( request: Request, resolvedUserId: string | undefined ): Promise { + if (!env.ADMIN_DASHBOARD_ENABLED) { + return resolveImpersonationState({ + impersonatedUserId: undefined, + viewingAsUser: undefined, + resolvedUserId, + }); + } + const session = await getImpersonationSession(request); return resolveImpersonationState({ diff --git a/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts b/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts index 01bdb6d9b53..8698b56bcd1 100644 --- a/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/dashboardBuilder.server.ts @@ -5,6 +5,7 @@ import { json, redirect } from "@remix-run/server-runtime"; import type { RbacAbility } from "@trigger.dev/rbac"; +import { env } from "~/env.server"; import { rbac } from "~/services/rbac.server"; import { getUserId } from "~/services/session.server"; import { permissionDeniedResponse } from "~/utils/permissionDenied"; @@ -23,7 +24,7 @@ function loginRedirectFor(request: Request, override?: string): Response { function isAuthorized(ability: RbacAbility, authorization: AuthorizationOption): boolean { if ("requireSuper" in authorization) { - return ability.canSuper(); + return env.ADMIN_DASHBOARD_ENABLED && ability.canSuper(); } return ability.can(authorization.action, authorization.resource); } diff --git a/apps/webapp/app/services/routeBuilders/permissions.server.ts b/apps/webapp/app/services/routeBuilders/permissions.server.ts index 37a70272c17..b94c7a25fac 100644 --- a/apps/webapp/app/services/routeBuilders/permissions.server.ts +++ b/apps/webapp/app/services/routeBuilders/permissions.server.ts @@ -1,4 +1,5 @@ import type { RbacAbility, RbacResource } from "@trigger.dev/rbac"; +import { env } from "~/env.server"; /** * A single permission check, mirroring the `authorization` option the @@ -32,7 +33,9 @@ export function checkPermissions( if (!Object.hasOwn(checks, key)) continue; const check = checks[key]; result[key] = - "requireSuper" in check ? ability.canSuper() : ability.can(check.action, check.resource); + "requireSuper" in check + ? env.ADMIN_DASHBOARD_ENABLED && ability.canSuper() + : ability.can(check.action, check.resource); } return result; } diff --git a/apps/webapp/test/impersonationDisabled.test.ts b/apps/webapp/test/impersonationDisabled.test.ts new file mode 100644 index 00000000000..233ef533f47 --- /dev/null +++ b/apps/webapp/test/impersonationDisabled.test.ts @@ -0,0 +1,92 @@ +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { env } from "~/env.server"; +import { clearImpersonation, redirectWithImpersonation } from "~/models/admin.server"; +import { + commitImpersonationSession, + getImpersonationId, + getImpersonationState, + getRawImpersonationId, + setImpersonationId, +} from "~/services/impersonation.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +// ADMIN_DASHBOARD_ENABLED=false: starting 404s, cookies resolve to nothing, +// stopping still works so lingering sessions can be terminated. +describe("impersonation disabled", () => { + postgresTest("the flag defaults to enabled", async () => { + // Flipping the default would kill the admin dashboard on every existing deployment. + expect(env.ADMIN_DASHBOARD_ENABLED).toBe(true); + }); + + postgresTest("starting impersonation 404s and cookies are inert", async ({ prisma }) => { + const admin = await prisma.user.create({ + data: { + email: `admin-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + admin: true, + }, + }); + const target = await prisma.user.create({ + data: { + email: `target-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: true, + }, + }); + + // A cookie minted while the flag was on, e.g. carried over or replayed. + const session = await setImpersonationId(target.id, new Request("http://localhost:3030/admin")); + const cookie = await commitImpersonationSession(session); + const requestWithCookie = () => + new Request("http://localhost:3030/", { headers: { Cookie: cookie } }); + + expect(await getImpersonationId(requestWithCookie())).toBe(target.id); + // resolvedUserId must be the impersonated id or the state is false even + // with the flag on, making the disabled assertion below vacuous. + const enabledState = await getImpersonationState(requestWithCookie(), target.id); + expect(enabledState.isImpersonating).toBe(true); + + const original = env.ADMIN_DASHBOARD_ENABLED; + // @ts-expect-error deliberately flipping the parsed env for the test + env.ADMIN_DASHBOARD_ENABLED = false; + try { + await expect( + redirectWithImpersonation( + new Request("http://localhost:3030/admin/impersonate", { method: "POST" }), + target.id, + "/", + { id: admin.id, admin: true }, + prisma + ) + ).rejects.toMatchObject({ status: 404 }); + + // No audit log: the gate fires before anything is recorded. + expect(await prisma.impersonationAuditLog.count()).toBe(0); + + expect(await getImpersonationId(requestWithCookie())).toBeUndefined(); + const disabledState = await getImpersonationState(requestWithCookie(), target.id); + expect(disabledState.isImpersonating).toBe(false); + + // The ungated reader still sees the cookie (stop/scrub paths need it). + expect(await getRawImpersonationId(requestWithCookie())).toBe(target.id); + + // Stopping works with the flag off and clears the cookie. + const response = await clearImpersonation(requestWithCookie(), "/"); + const setCookie = response.headers.get("set-cookie"); + expect(setCookie).toContain("__impersonate="); + const clearedRequest = new Request("http://localhost:3030/", { + headers: { Cookie: setCookie!.split(";")[0] }, + }); + expect(await getRawImpersonationId(clearedRequest)).toBeUndefined(); + } finally { + // @ts-expect-error restore the parsed env + env.ADMIN_DASHBOARD_ENABLED = original; + } + }); +}); diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 6e89dd52aba..8d31694686e 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -184,6 +184,7 @@ mode: "wide" | `MACHINE_PRESETS_OVERRIDE_PATH` | No | — | Path to machine presets override file. See [machine overrides](/self-hosting/overview#machine-overrides). | | `APP_ENV` | No | `NODE_ENV` | App environment. Used for things like the title tag. | | `ADMIN_EMAILS` | No | — | Regex of user emails to automatically promote to admin on signup. Does not apply to existing users. | +| `ADMIN_DASHBOARD_ENABLED` | No | 1 | Set to anything other than `1` or `true` to disable the admin dashboard and user impersonation on this instance. | | `EVENT_LOOP_MONITOR_ENABLED` | No | 1 | Node.js event loop lag monitor. | ## Multi-Provider Object Storage From 47ff76d72722e2204627b4ee47716b1aecd8a7a6 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 25 Aug 2026 14:49:10 +0100 Subject: [PATCH 13/28] feat(webapp,clickhouse): return an actionable error instead of a 500 when a runs list query is too expensive (#4773) ## Summary When a runs list query is too expensive to complete, it now fails with a clear, actionable error instead of a generic 500. Previously, a runs list query that exceeded ClickHouse resource limits threw an opaque error. On the public `runs.list` API that surfaced as a retryable 500, so a customer task calling it would keep retrying a query that could never succeed. On the dashboard it rendered as a generic error page with no hint about what to do. ## Fix The ClickHouse client now tags resource-limit failures (memory, time, rows, bytes) with their error type, and the runs repository maps those to a dedicated `RunsListQueryError` (HTTP 422). - `runs.list` API returns 422 with a message telling the user to narrow their `created_at` range, plus an `x-should-retry: false` header so the SDK does not retry it. - The dashboard runs list (and the errors, scheduled, standard-task, agents, and webhooks list views) render a shared error state with the same guidance, so a too-broad time filter is recoverable by the user. --- .../runs-list-query-limit-error.md | 6 + .../components/runs/v3/RunsListErrorState.tsx | 28 + .../v3/ErrorGroupPresenter.server.ts | 5 +- .../route.tsx | 26 +- .../route.tsx | 22 +- .../route.tsx | 14 +- .../route.tsx | 26 +- .../route.tsx | 26 +- .../route.tsx | 26 +- apps/webapp/app/routes/api.v1.runs.ts | 25 +- .../clickhouseRunsRepository.server.ts | 23 +- .../runsRepository/runsRepository.server.ts | 27 + .../test/clickhouseQueryMetrics.test.ts | 93 +++ apps/webapp/test/runsListQueryError.test.ts | 52 ++ .../clickhouse/src/client/client.ts | 752 +++++++++++------- .../clickhouse/src/client/errors.ts | 31 +- internal-packages/clickhouse/src/index.ts | 7 +- 17 files changed, 813 insertions(+), 376 deletions(-) create mode 100644 .server-changes/runs-list-query-limit-error.md create mode 100644 apps/webapp/app/components/runs/v3/RunsListErrorState.tsx create mode 100644 apps/webapp/test/clickhouseQueryMetrics.test.ts create mode 100644 apps/webapp/test/runsListQueryError.test.ts diff --git a/.server-changes/runs-list-query-limit-error.md b/.server-changes/runs-list-query-limit-error.md new file mode 100644 index 00000000000..1bd16673ef8 --- /dev/null +++ b/.server-changes/runs-list-query-limit-error.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +When a runs list or runs.list API request spans too much data to complete, it now returns a clear, actionable error asking you to narrow the time range, instead of failing with a generic error. diff --git a/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx b/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx new file mode 100644 index 00000000000..85f75e00602 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/RunsListErrorState.tsx @@ -0,0 +1,28 @@ +import { Callout } from "~/components/primitives/Callout"; + +/** + * Error state for a runs list that failed to load. Shown as the `errorElement` of the deferred + * runs-list data. The most common recoverable cause is a query that was too expensive over a broad + * time range (see `RunsListQueryError`), so the copy guides narrowing the range; a refresh covers + * transient failures. The precise reason is not shown because Remix scrubs thrown error messages in + * production. + */ +export function RunsListErrorState() { + return ( +
+ + We couldn't load these runs. If you're filtering over a broad time range, try narrowing it, + then refresh to try again. + +
+ ); +} + +/** + * Renders nothing. Used as the `errorElement` for secondary awaits of the same runs-list promise + * (e.g. the pagination controls), so a rejection is handled locally there and does not bubble to + * the route error boundary. The primary awaits render {@link RunsListErrorState}. + */ +export function RunsListErrorStateNoop() { + return null; +} diff --git a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts index 38149abe5e7..c2015c20bb4 100644 --- a/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ErrorGroupPresenter.server.ts @@ -87,7 +87,8 @@ export class ErrorGroupPresenter extends BasePresenter { constructor( private readonly replica: PrismaClientOrTransaction, private readonly logsClickhouse: ClickHouse, - private readonly clickhouse: ClickHouse + private readonly clickhouse: ClickHouse, + private readonly runsListClickhouse: ClickHouse ) { super(undefined, replica); } @@ -409,7 +410,7 @@ export class ErrorGroupPresenter extends BasePresenter { columns?: RunColumnsSelect; } ): Promise { - const runListPresenter = new NextRunListPresenter(this.replica, this.clickhouse); + const runListPresenter = new NextRunListPresenter(this.replica, this.runsListClickhouse); const result = await runListPresenter.call(organizationId, environmentId, { userId: options.userId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx index e0354185363..2fd312e3257 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx @@ -22,6 +22,11 @@ import { Paragraph } from "~/components/primitives/Paragraph"; import * as Property from "~/components/primitives/PropertyTable"; import { Spinner } from "~/components/primitives/Spinner"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import { + RunsListErrorState, + RunsListErrorStateNoop, +} from "~/components/runs/v3/RunsListErrorState"; +import { RunsListQueryError } from "~/services/runsRepository/runsRepository.server"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { SessionsTable } from "~/components/sessions/v1/SessionsTable"; @@ -92,10 +97,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new AgentDetailPresenter($replica, clickhouse); const agent = await presenter.findAgent({ @@ -154,7 +159,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies AgentActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -166,7 +171,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { direction, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); const sessionList = new SessionListPresenter($replica, clickhouse) .call(project.organizationId, environment.id, { @@ -341,7 +351,7 @@ export default function Page() { <> - + }> {(list) => (list ? : null)} @@ -395,7 +405,7 @@ function AgentContentArea({ ) : ( }> - }> + }> {(list) => list ? ( { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const [logsClickhouseClient, clickhouseClient] = await Promise.all([ + const [logsClickhouseClient, clickhouseClient, runsListClickhouseClient] = await Promise.all([ clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "logs"), clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "runsList"), ]); - const presenter = new ErrorGroupPresenter($replica, logsClickhouseClient, clickhouseClient); + const presenter = new ErrorGroupPresenter( + $replica, + logsClickhouseClient, + clickhouseClient, + runsListClickhouseClient + ); const detailPromise = presenter .call(project.organizationId, environment.id, { @@ -393,16 +400,7 @@ export default function Page() { } > - - - Unable to load error details. Please refresh the page or try again in a moment. - - - } - > + }> {(result) => { if ("error" in result) { return ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx index 921e429700c..0abcd9444ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx @@ -74,7 +74,7 @@ import { import { throwNotFound } from "~/utils/httpErrors"; import { ListPagination } from "../../components/ListPagination"; import { CreateBulkActionInspector } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { Callout } from "~/components/primitives/Callout"; +import { RunsListErrorState } from "~/components/runs/v3/RunsListErrorState"; import { isRunsListLoading, RUNS_BULK_INSPECTOR_OPEN_VALUE, @@ -208,17 +208,7 @@ export default function Page() { } > - - - Unable to load your task runs. Please refresh the page or try again in a - moment. - - - } - > + }> {(list) => { return ( { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const taskPresenter = new TaskDetailPresenter($replica, clickhouse); const task = await taskPresenter.findTask({ @@ -211,7 +216,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => null); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -224,7 +229,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { includeHasAnyRuns: true, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); return typeddefer({ task, @@ -375,14 +385,14 @@ export default function Page() { ) : null} - + }> {(list) => (list ? : null)}
}> - }> + }> {(list) => list ? ( { const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; const versions = url.searchParams.getAll("versions").filter((v) => v.length > 0); - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new TaskDetailPresenter($replica, clickhouse); const task = await presenter.findTask({ @@ -153,7 +158,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies TaskActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -167,7 +172,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { includeHasAnyRuns: true, columns: getRunColumnsForSelect(request), }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); return typeddefer({ task, @@ -271,14 +281,14 @@ export default function Page() { ) : null} - + }> {(list) => (list ? : null)}
}> - }> + }> {(list) => list ? ( { const runsDirectionRaw = url.searchParams.get("runsDirection") ?? undefined; const runsDirection = runsDirectionRaw ? DirectionSchema.parse(runsDirectionRaw) : undefined; - const clickhouse = await clickhouseFactory.getClickhouseForOrganization( - project.organizationId, - "standard" - ); + const [clickhouse, runsListClickhouse] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"), + clickhouseFactory.getClickhouseForOrganization(project.organizationId, "runsList"), + ]); const presenter = new WebhookDetailPresenter($replica, clickhouse); const webhook = await presenter.findWebhook({ @@ -156,7 +161,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => ({ data: [], statuses: [] }) satisfies WebhookActivity); - const runList = new NextRunListPresenter($replica, clickhouse) + const runList = new NextRunListPresenter($replica, runsListClickhouse) .call(project.organizationId, environment.id, { userId, projectId: project.id, @@ -167,7 +172,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { cursor: runsCursor, direction: runsDirection, }) - .catch(() => null); + .catch((error) => { + if (error instanceof RunsListQueryError) { + throw error; + } + return null; + }); const deliveriesList = presenter .listDeliveries({ @@ -329,7 +339,7 @@ export default function Page() { ) : ( - + }> {(list) => list ? ( ) : ( }> - }> + }> {(list) => list ? (
diff --git a/apps/webapp/app/routes/api.v1.runs.ts b/apps/webapp/app/routes/api.v1.runs.ts index dca246a0c24..a9f2b8b4d2c 100644 --- a/apps/webapp/app/routes/api.v1.runs.ts +++ b/apps/webapp/app/routes/api.v1.runs.ts @@ -8,6 +8,7 @@ import { createLoaderApiRoute, everyResource, } from "~/services/routeBuilders/apiBuilder.server"; +import { RunsListQueryError } from "~/services/runsRepository/runsRepository.server"; export const loader = createLoaderApiRoute( { @@ -40,13 +41,23 @@ export const loader = createLoaderApiRoute( }, async ({ searchParams, authentication, apiVersion }) => { const presenter = new ApiRunListPresenter(); - const result = await presenter.call( - authentication.environment.project, - searchParams, - apiVersion, - authentication.environment - ); + try { + const result = await presenter.call( + authentication.environment.project, + searchParams, + apiVersion, + authentication.environment + ); - return json(result); + return json(result); + } catch (error) { + if (error instanceof RunsListQueryError) { + return json( + { error: error.message }, + { status: error.status, headers: { "x-should-retry": "false" } } + ); + } + throw error; + } } ); diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 4981dd19c43..02b3e014466 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -1,4 +1,4 @@ -import { type ClickhouseQueryBuilder } from "@internal/clickhouse"; +import { type ClickhouseQueryBuilder, isClickhouseResourceLimitError } from "@internal/clickhouse"; import { ErrorId, RunId } from "@trigger.dev/core/v3/isomorphic"; import { type FilterRunsOptions, @@ -10,6 +10,7 @@ import { type RunsRepositoryOptions, type TagListOptions, convertRunListInputOptionsToFilterRunsOptions, + RunsListQueryError, } from "./runsRepository.server"; import parseDuration from "parse-duration"; import { decodeRunsCursor, encodeRunsCursor } from "./runsCursor.server"; @@ -19,6 +20,18 @@ import { type PrismaClientOrTransaction } from "~/db.server"; import { boundedIn, type Prisma } from "@trigger.dev/database"; type RunCursorRow = { runId: string; createdAt: number }; +/** + * Re-throws a runs-list query error, converting a ClickHouse resource-limit rejection (execution + * time or memory) into a typed {@link RunsListQueryError} so callers can surface an actionable 4xx + * instead of an opaque 500. Any other error is re-thrown unchanged. + */ +function rethrowRunsListQueryError(queryError: unknown): never { + if (isClickhouseResourceLimitError(queryError)) { + throw new RunsListQueryError(undefined, { cause: queryError }); + } + throw queryError; +} + /** * Default hydrate select for the runs list, used when a caller does not derive * one from the visible columns (bulk actions, the live poll). Kept in sync with @@ -102,7 +115,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return (result?.length ?? 0) > 0; @@ -166,7 +179,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return result.map((row) => ({ runId: row.run_id, createdAt: row.created_at_ms })); @@ -349,7 +362,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } if (result.length === 0) { @@ -402,7 +415,7 @@ export class ClickHouseRunsRepository implements IRunsRepository { const [queryError, result] = await queryBuilder.execute(); if (queryError) { - throw queryError; + rethrowRunsListQueryError(queryError); } return { diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 0b1049125dd..1aeeb96dbc1 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -13,6 +13,33 @@ import { runStore as defaultRunStore } from "~/v3/runStore.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server"; +/** + * User-facing message when a runs-list query exceeds a ClickHouse resource limit. It tells the + * caller how to recover (a narrower time range restores partition pruning), and is safe to show + * on the dashboard and return from the public API. + */ +const RUNS_LIST_QUERY_TOO_EXPENSIVE_MESSAGE = + "This query was too expensive to run over the selected time range. Narrow the time window (a shorter period, or a smaller createdAt from/to range) and try again."; + +/** + * Thrown when a runs-list ClickHouse query hits a server-side resource limit (execution time or + * memory). It is the caller's query being too broad, not a service fault, so it carries a 4xx + * status and a recovery message rather than surfacing as a 500. + */ +export class RunsListQueryError extends Error { + public readonly name = "RunsListQueryError"; + public readonly status = 422; + constructor( + message: string = RUNS_LIST_QUERY_TOO_EXPENSIVE_MESSAGE, + options?: { cause?: unknown } + ) { + super(message); + if (options?.cause !== undefined) { + this.cause = options.cause; + } + } +} + export type RunsRepositoryOptions = { clickhouse: ClickHouse; prisma: PrismaClientOrTransaction; diff --git a/apps/webapp/test/clickhouseQueryMetrics.test.ts b/apps/webapp/test/clickhouseQueryMetrics.test.ts new file mode 100644 index 00000000000..29e4bc85835 --- /dev/null +++ b/apps/webapp/test/clickhouseQueryMetrics.test.ts @@ -0,0 +1,93 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; +import { createInMemoryMetrics } from "./utils/tracing"; +import { histogramCount, latestMetrics, metricSum } from "./otlpMetrics.helpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +describe("clickhouse query metrics", () => { + containerTest( + "records duration + read_rows on success and an error metric with the ClickHouse error type", + async ({ clickhouseContainer, prisma }) => { + const ctx = await seedParents(prisma, "chm"); + const run = await createRun(prisma, ctx, { friendlyId: "run_chm" }); + + const seedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-seed", + }); + await insertTaskRunV2Rows(seedClient, [{ ...run, createdAt: new Date() }]); + + const listArgs = { + page: { size: 10 } as const, + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const okMetrics = createInMemoryMetrics(); + const okClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-ok", + meter: okMetrics.meter, + }); + const okRepo = new RunsRepository({ prisma, clickhouse: okClient }); + const result = await okRepo.listRuns(listArgs); + expect(result.runs.map((r) => r.friendlyId)).toEqual(["run_chm"]); + + await vi.waitFor( + async () => { + const rm = await latestMetrics(okMetrics); + expect( + histogramCount(rm, "clickhouse.query.duration", { + client: "clickhouse-metrics-ok", + status: "ok", + }) + ).toBeGreaterThanOrEqual(1); + }, + { timeout: 5000, interval: 50 } + ); + const okRm = await latestMetrics(okMetrics); + expect( + histogramCount(okRm, "clickhouse.query.read_rows", { client: "clickhouse-metrics-ok" }) + ).toBeGreaterThanOrEqual(1); + expect( + histogramCount(okRm, "clickhouse.query.memory_usage", { client: "clickhouse-metrics-ok" }) + ).toBeGreaterThanOrEqual(1); + await okMetrics.shutdown(); + + const errMetrics = createInMemoryMetrics(); + const cappedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "clickhouse-metrics-capped", + clickhouseSettings: { max_memory_usage: "1" }, + meter: errMetrics.meter, + }); + const errRepo = new RunsRepository({ prisma, clickhouse: cappedClient }); + await expect(errRepo.listRuns(listArgs)).rejects.toThrow(); + + await vi.waitFor( + async () => { + const rm = await latestMetrics(errMetrics); + expect( + metricSum(rm, "clickhouse.query.errors", { + client: "clickhouse-metrics-capped", + error_type: "MEMORY_LIMIT_EXCEEDED", + }) + ).toBeGreaterThanOrEqual(1); + }, + { timeout: 5000, interval: 50 } + ); + await errMetrics.shutdown(); + } + ); +}); diff --git a/apps/webapp/test/runsListQueryError.test.ts b/apps/webapp/test/runsListQueryError.test.ts new file mode 100644 index 00000000000..89e0486bf72 --- /dev/null +++ b/apps/webapp/test/runsListQueryError.test.ts @@ -0,0 +1,52 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { + RunsListQueryError, + RunsRepository, +} from "~/services/runsRepository/runsRepository.server"; +import { + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +vi.setConfig({ testTimeout: 90_000 }); + +describe("runs list query error handling", () => { + containerTest( + "a ClickHouse resource-limit error surfaces as RunsListQueryError", + async ({ clickhouseContainer, prisma }) => { + const ctx = await seedParents(prisma, "qerr"); + const run = await createRun(prisma, ctx, { friendlyId: "run_qerr" }); + + const seedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-query-error-seed", + }); + await insertTaskRunV2Rows(seedClient, [{ ...run, createdAt: new Date() }]); + + const listArgs = { + page: { size: 10 } as const, + organizationId: ctx.organizationId, + projectId: ctx.projectId, + environmentId: ctx.environmentId, + }; + + const cappedClient = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-list-query-error-capped", + clickhouseSettings: { max_memory_usage: "1" }, + }); + const capped = new RunsRepository({ prisma, clickhouse: cappedClient }); + await expect(capped.listRuns(listArgs)).rejects.toBeInstanceOf(RunsListQueryError); + await expect(capped.countRuns(listArgs)).rejects.toBeInstanceOf(RunsListQueryError); + + const ok = new RunsRepository({ prisma, clickhouse: seedClient }); + const result = await ok.listRuns(listArgs); + expect(result.runs.map((r) => r.friendlyId)).toEqual(["run_qerr"]); + } + ); +}); diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index 9949081d504..96db1f4a529 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -7,8 +7,8 @@ import { type BaseQueryParams, type InsertResult, } from "@clickhouse/client"; -import type { Span, Tracer } from "@internal/tracing"; -import { recordSpanError, startSpan, trace } from "@internal/tracing"; +import type { Counter, Histogram, Meter, Span, Tracer, UpDownCounter } from "@internal/tracing"; +import { getMeter, recordSpanError, startSpan, trace } from "@internal/tracing"; import { flattenAttributes, tryCatch, type Result } from "@trigger.dev/core/v3"; import { z } from "zod"; import { InsertError, QueryError } from "./errors.js"; @@ -43,6 +43,7 @@ export type ClickhouseConfig = { httpAgent?: HttpAgent | HttpsAgent; clickhouseSettings?: ClickHouseSettings; logger?: Logger; + meter?: Meter; maxOpenConnections?: number; requestTimeoutMs?: number; logLevel?: LogLevel; @@ -57,11 +58,49 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { private readonly tracer: Tracer; private readonly name: string; private readonly logger: Logger; + private readonly meter: Meter; + private readonly queryInFlight: UpDownCounter; + private readonly queryDuration: Histogram; + private readonly queryServerDuration: Histogram; + private readonly queryReadRows: Histogram; + private readonly queryReadBytes: Histogram; + private readonly queryMemoryUsage: Histogram; + private readonly queryErrors: Counter; constructor(config: ClickhouseConfig) { this.name = config.name; this.logger = config.logger ?? new Logger("ClickhouseClient", config.logLevel ?? "info"); + this.meter = config.meter ?? getMeter("clickhouse"); + this.queryInFlight = this.meter.createUpDownCounter("clickhouse.query.in_flight", { + description: "Concurrent in-flight ClickHouse queries per client, a pool-saturation signal", + }); + this.queryDuration = this.meter.createHistogram("clickhouse.query.duration", { + description: + "Wall-clock ClickHouse query duration, includes client-side connection-pool wait", + unit: "ms", + }); + this.queryServerDuration = this.meter.createHistogram("clickhouse.query.server_duration", { + description: "Server-side ClickHouse query duration from the x-clickhouse-summary elapsed_ns", + unit: "ms", + }); + this.queryReadRows = this.meter.createHistogram("clickhouse.query.read_rows", { + description: "Rows read by a ClickHouse query, from the x-clickhouse-summary header", + unit: "{row}", + }); + this.queryReadBytes = this.meter.createHistogram("clickhouse.query.read_bytes", { + description: "Bytes read by a ClickHouse query, from the x-clickhouse-summary header", + unit: "By", + }); + this.queryMemoryUsage = this.meter.createHistogram("clickhouse.query.memory_usage", { + description: "Peak memory used by a ClickHouse query, from the x-clickhouse-summary header", + unit: "By", + }); + this.queryErrors = this.meter.createCounter("clickhouse.query.errors", { + description: + "ClickHouse query errors by type, e.g. MEMORY_LIMIT_EXCEEDED or TIMEOUT_EXCEEDED", + }); + this.client = createClient({ url: config.url, keep_alive: config.keepAlive, @@ -87,6 +126,40 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { await this.client.close(); } + private recordQueryMetrics( + operation: string, + startedAt: number, + result: { errorType?: string; summary?: Record } + ) { + const attributes = { client: this.name, operation }; + this.queryDuration.record(performance.now() - startedAt, { + ...attributes, + status: result.errorType ? "error" : "ok", + }); + if (result.errorType) { + this.queryErrors.add(1, { ...attributes, error_type: result.errorType }); + } + const summary = result.summary; + if (summary) { + const elapsedNs = Number(summary.elapsed_ns); + if (Number.isFinite(elapsedNs) && elapsedNs > 0) { + this.queryServerDuration.record(elapsedNs / 1_000_000, attributes); + } + const readRows = Number(summary.read_rows); + if (Number.isFinite(readRows)) { + this.queryReadRows.record(readRows, attributes); + } + const readBytes = Number(summary.read_bytes); + if (Number.isFinite(readBytes)) { + this.queryReadBytes.record(readBytes, attributes); + } + const memoryUsage = Number(summary.memory_usage); + if (Number.isFinite(memoryUsage) && memoryUsage > 0) { + this.queryMemoryUsage.record(memoryUsage, attributes); + } + } + } + public query, TOut extends z.ZodSchema>(req: { /** * The name of the operation. @@ -117,124 +190,147 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "query", async (span) => { - this.logger.debug("Querying clickhouse", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const validParams = req.params?.safeParse(params); - - if (validParams?.error) { - recordSpanError(span, validParams.error); - - this.logger.error("Error parsing query params", { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "query", + async (span): Promise[], QueryError>> => { + this.logger.debug("Querying clickhouse", { name: req.name, - error: validParams.error, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, }); - return [ - new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { - query: req.query, - }), - null, - ]; - } - - let unparsedRows: Array = []; - - const [clickhouseError, res] = await tryCatch( - this.client.query({ - query: req.query, - query_params: validParams?.data, - format: "JSONEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); - - if (clickhouseError) { - const errorLogFields = { - name: req.name, - error: clickhouseError, - query: req.query, - params, - queryId, - }; + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - this.logger.error("Error querying clickhouse", errorLogFields); + const validParams = req.params?.safeParse(params); - recordClickhouseError(span, clickhouseError); + if (validParams?.error) { + recordSpanError(span, validParams.error); - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + this.logger.error("Error parsing query params", { + name: req.name, + error: validParams.error, query: req.query, - }), - null, - ]; - } + params, + queryId, + }); + + return [ + new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { + query: req.query, + }), + null, + ]; + } - unparsedRows = await res.json(); + let unparsedRows: Array = []; - span.setAttributes({ - "clickhouse.query_id": res.query_id, - ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), - }); + const [clickhouseError, res] = await tryCatch( + this.client.query({ + query: req.query, + query_params: validParams?.data, + format: "JSONEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + this.logger.error("Error querying clickhouse", errorLogFields); + + recordClickhouseError(span, clickhouseError); + + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - const summaryHeader = res.response_headers["x-clickhouse-summary"]; + unparsedRows = await res.json(); - if (typeof summaryHeader === "string") { span.setAttributes({ - ...flattenAttributes(JSON.parse(summaryHeader), "clickhouse.summary"), + "clickhouse.query_id": res.query_id, + ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), }); - } - const parsed = z.array(req.schema).safeParse(unparsedRows); + const summaryHeader = res.response_headers["x-clickhouse-summary"]; - if (parsed.error) { - this.logger.error("Error parsing clickhouse query result", { - name: req.name, - error: parsed.error, - query: req.query, - params, - queryId, - }); + if (typeof summaryHeader === "string") { + summary = JSON.parse(summaryHeader); + span.setAttributes({ + ...flattenAttributes(summary, "clickhouse.summary"), + }); + } - const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { - query: req.query, - }); + const parsed = z.array(req.schema).safeParse(unparsedRows); - recordSpanError(span, queryError); + if (parsed.error) { + this.logger.error("Error parsing clickhouse query result", { + name: req.name, + error: parsed.error, + query: req.query, + params, + queryId, + }); - return [queryError, null]; - } + const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { + query: req.query, + }); - span.setAttributes({ - "clickhouse.rows": unparsedRows.length, - }); + recordSpanError(span, queryError); + + return [queryError, null]; + } - return [null, parsed.data]; + span.setAttributes({ + "clickhouse.rows": unparsedRows.length, + }); + + return [null, parsed.data]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } @@ -278,163 +374,188 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryWithStatsFunction, z.output> { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "queryWithStats", async (span) => { - this.logger.debug("Querying clickhouse with stats", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const validParams = req.params?.safeParse(params); - - if (validParams?.error) { - recordSpanError(span, validParams.error); - - this.logger.error("Error parsing query params", { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "queryWithStats", + async ( + span + ): Promise[]; stats: QueryStats }, QueryError>> => { + this.logger.debug("Querying clickhouse with stats", { name: req.name, - error: validParams.error, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, }); - return [ - new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { - query: req.query, - }), - null, - ]; - } - - let unparsedRows: Array = []; + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - const [clickhouseError, res] = await tryCatch( - this.client.query({ - query: req.query, - query_params: validParams?.data, - format: "JSONEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); + const validParams = req.params?.safeParse(params); - if (clickhouseError) { - const errorLogFields = { - ...req.logFields, - name: req.name, - error: clickhouseError, - query: req.query, - params, - queryId, - }; + if (validParams?.error) { + recordSpanError(span, validParams.error); - switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { - case "quota": - this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); - break; - case "invalid-sql": - this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); - break; - default: - this.logger.error("Error querying clickhouse", errorLogFields); + this.logger.error("Error parsing query params", { + name: req.name, + error: validParams.error, + query: req.query, + params, + queryId, + }); + + return [ + new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, { + query: req.query, + }), + null, + ]; } - recordClickhouseError(span, clickhouseError); + let unparsedRows: Array = []; - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + const [clickhouseError, res] = await tryCatch( + this.client.query({ query: req.query, - }), - null, - ]; - } + query_params: validParams?.data, + format: "JSONEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + ...req.logFields, + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); + } - unparsedRows = await res.json(); + recordClickhouseError(span, clickhouseError); - span.setAttributes({ - "clickhouse.query_id": res.query_id, - ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), - }); + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - // Parse the summary header to get stats - const summaryHeader = res.response_headers["x-clickhouse-summary"]; - let stats: QueryStats = { - read_rows: "0", - read_bytes: "0", - written_rows: "0", - written_bytes: "0", - total_rows_to_read: "0", - result_rows: "0", - result_bytes: "0", - elapsed_ns: "0", - byte_seconds: "0", - }; + unparsedRows = await res.json(); - if (typeof summaryHeader === "string") { - const parsedSummary = JSON.parse(summaryHeader); - this.logger.debug("parsedSummary", parsedSummary); - const readBytes = parsedSummary.read_bytes ? parseInt(parsedSummary.read_bytes, 10) : 0; - const elapsedNs = parsedSummary.elapsed_ns ? parseInt(parsedSummary.elapsed_ns, 10) : 0; - const elapsedSeconds = elapsedNs / 1_000_000_000; - const byteSeconds = elapsedSeconds > 0 ? readBytes / elapsedSeconds : 0; - stats = { - read_rows: parsedSummary.read_rows ?? "0", - read_bytes: parsedSummary.read_bytes ?? "0", - written_rows: parsedSummary.written_rows ?? "0", - written_bytes: parsedSummary.written_bytes ?? "0", - total_rows_to_read: parsedSummary.total_rows_to_read ?? "0", - result_rows: parsedSummary.result_rows ?? "0", - result_bytes: parsedSummary.result_bytes ?? "0", - elapsed_ns: parsedSummary.elapsed_ns ?? "0", - byte_seconds: byteSeconds.toString(), - }; span.setAttributes({ - ...flattenAttributes(parsedSummary, "clickhouse.summary"), + "clickhouse.query_id": res.query_id, + ...flattenAttributes(res.response_headers, "clickhouse.response_headers"), }); - } - const parsed = z.array(req.schema).safeParse(unparsedRows); + // Parse the summary header to get stats + const summaryHeader = res.response_headers["x-clickhouse-summary"]; + let stats: QueryStats = { + read_rows: "0", + read_bytes: "0", + written_rows: "0", + written_bytes: "0", + total_rows_to_read: "0", + result_rows: "0", + result_bytes: "0", + elapsed_ns: "0", + byte_seconds: "0", + }; - if (parsed.error) { - this.logger.error("Error parsing clickhouse query result", { - name: req.name, - error: parsed.error, - query: req.query, - params, - queryId, - }); + if (typeof summaryHeader === "string") { + const parsedSummary = JSON.parse(summaryHeader); + summary = parsedSummary; + this.logger.debug("parsedSummary", parsedSummary); + const readBytes = parsedSummary.read_bytes ? parseInt(parsedSummary.read_bytes, 10) : 0; + const elapsedNs = parsedSummary.elapsed_ns ? parseInt(parsedSummary.elapsed_ns, 10) : 0; + const elapsedSeconds = elapsedNs / 1_000_000_000; + const byteSeconds = elapsedSeconds > 0 ? readBytes / elapsedSeconds : 0; + stats = { + read_rows: parsedSummary.read_rows ?? "0", + read_bytes: parsedSummary.read_bytes ?? "0", + written_rows: parsedSummary.written_rows ?? "0", + written_bytes: parsedSummary.written_bytes ?? "0", + total_rows_to_read: parsedSummary.total_rows_to_read ?? "0", + result_rows: parsedSummary.result_rows ?? "0", + result_bytes: parsedSummary.result_bytes ?? "0", + elapsed_ns: parsedSummary.elapsed_ns ?? "0", + byte_seconds: byteSeconds.toString(), + }; + span.setAttributes({ + ...flattenAttributes(parsedSummary, "clickhouse.summary"), + }); + } - const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { - query: req.query, - }); + const parsed = z.array(req.schema).safeParse(unparsedRows); + + if (parsed.error) { + this.logger.error("Error parsing clickhouse query result", { + name: req.name, + error: parsed.error, + query: req.query, + params, + queryId, + }); - recordSpanError(span, queryError); + const queryError = new QueryError(generateErrorMessage(parsed.error.issues), { + query: req.query, + }); - return [queryError, null]; - } + recordSpanError(span, queryError); - span.setAttributes({ - "clickhouse.rows": unparsedRows.length, - }); + return [queryError, null]; + } + + span.setAttributes({ + "clickhouse.rows": unparsedRows.length, + }); - return [null, { rows: parsed.data, stats }]; + return [null, { rows: parsed.data, stats }]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } @@ -446,103 +567,126 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }): ClickhouseQueryFunction { return async (params, options) => { const queryId = randomUUID(); - - return await startSpan(this.tracer, "queryFast", async (span) => { - this.logger.debug("Querying clickhouse fast", { - name: req.name, - query: req.query.replace(/\s+/g, " "), - params, - settings: req.settings, - attributes: options?.attributes, - queryId, - }); - - span.setAttributes({ - "clickhouse.clientName": this.name, - "clickhouse.operationName": req.name, - "clickhouse.queryId": queryId, - ...flattenAttributes(req.settings, "clickhouse.settings"), - ...flattenAttributes(options?.attributes), - }); - - const [clickhouseError, resultSet] = await tryCatch( - this.client.query({ - query: req.query, - query_params: params, - format: "JSONCompactEachRow", - query_id: queryId, - ...options?.params, - clickhouse_settings: { - ...req.settings, - ...options?.params?.clickhouse_settings, - }, - }) - ); - - if (clickhouseError) { - const errorLogFields = { + const startedAt = performance.now(); + this.queryInFlight.add(1, { client: this.name }); + let summary: Record | undefined; + + const result = await startSpan( + this.tracer, + "queryFast", + async (span): Promise> => { + this.logger.debug("Querying clickhouse fast", { name: req.name, - error: clickhouseError, - query: req.query, + query: req.query.replace(/\s+/g, " "), params, + settings: req.settings, + attributes: options?.attributes, queryId, - }; - - this.logger.error("Error querying clickhouse", errorLogFields); + }); - recordClickhouseError(span, clickhouseError); + span.setAttributes({ + "clickhouse.clientName": this.name, + "clickhouse.operationName": req.name, + "clickhouse.queryId": queryId, + ...flattenAttributes(req.settings, "clickhouse.settings"), + ...flattenAttributes(options?.attributes), + }); - return [ - new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, { + const [clickhouseError, resultSet] = await tryCatch( + this.client.query({ query: req.query, - }), - null, - ]; - } - - span.setAttributes({ - "clickhouse.query_id": resultSet.query_id, - ...flattenAttributes(resultSet.response_headers, "clickhouse.response_headers"), - }); - - const summaryHeader = resultSet.response_headers["x-clickhouse-summary"]; + query_params: params, + format: "JSONCompactEachRow", + query_id: queryId, + ...options?.params, + clickhouse_settings: { + ...req.settings, + ...options?.params?.clickhouse_settings, + }, + }) + ); + + if (clickhouseError) { + const errorLogFields = { + name: req.name, + error: clickhouseError, + query: req.query, + params, + queryId, + }; + + this.logger.error("Error querying clickhouse", errorLogFields); + + recordClickhouseError(span, clickhouseError); + + return [ + new QueryError( + `Unable to query clickhouse: ${clickhouseError.message}`, + { query: req.query }, + clickhouseError instanceof ClickHouseError ? clickhouseError.type : undefined + ), + null, + ]; + } - if (typeof summaryHeader === "string") { span.setAttributes({ - ...flattenAttributes(JSON.parse(summaryHeader), "clickhouse.summary"), + "clickhouse.query_id": resultSet.query_id, + ...flattenAttributes(resultSet.response_headers, "clickhouse.response_headers"), }); - } - const resultRows: Array = []; + const summaryHeader = resultSet.response_headers["x-clickhouse-summary"]; - for await (const rows of resultSet.stream()) { - if (rows.length === 0) { - continue; + if (typeof summaryHeader === "string") { + summary = JSON.parse(summaryHeader); + span.setAttributes({ + ...flattenAttributes(summary, "clickhouse.summary"), + }); } - for (const row of rows) { - const rowData = row.json() as any[]; + const resultRows: Array = []; - const hydratedRow: Record = {}; - for (let i = 0; i < req.columns.length; i++) { - const column = req.columns[i]; + for await (const rows of resultSet.stream()) { + if (rows.length === 0) { + continue; + } - if (typeof column === "string") { - hydratedRow[column] = rowData[i]; - } else { - hydratedRow[column.name] = rowData[i]; + for (const row of rows) { + const rowData = row.json() as any[]; + + const hydratedRow: Record = {}; + for (let i = 0; i < req.columns.length; i++) { + const column = req.columns[i]; + + if (typeof column === "string") { + hydratedRow[column] = rowData[i]; + } else { + hydratedRow[column.name] = rowData[i]; + } } + resultRows.push(hydratedRow as TOut); } - resultRows.push(hydratedRow as TOut); } - } - span.setAttributes({ - "clickhouse.rows": resultRows.length, - }); + span.setAttributes({ + "clickhouse.rows": resultRows.length, + }); - return [null, resultRows]; + return [null, resultRows]; + } + ) + .catch((error) => { + this.recordQueryMetrics(req.name, startedAt, { errorType: "exception" }); + throw error; + }) + .finally(() => this.queryInFlight.add(-1, { client: this.name })); + + this.recordQueryMetrics(req.name, startedAt, { + errorType: + result[0] instanceof QueryError ? (result[0].clickhouseErrorType ?? "other") : undefined, + summary, }); + + return result; }; } diff --git a/internal-packages/clickhouse/src/client/errors.ts b/internal-packages/clickhouse/src/client/errors.ts index ff0be4d0d54..dd4de178055 100644 --- a/internal-packages/clickhouse/src/client/errors.ts +++ b/internal-packages/clickhouse/src/client/errors.ts @@ -45,10 +45,39 @@ export class InsertError extends BaseError { export class QueryError extends BaseError<{ query: string }> { public readonly retry = true; public readonly name = QueryError.name; - constructor(message: string, context: { query: string }) { + /** + * The underlying ClickHouse error type (e.g. `TIMEOUT_EXCEEDED`) when the failure came from + * ClickHouse rejecting the query, else undefined. Lets callers distinguish a query that hit a + * server-side resource limit from an unexpected failure. + */ + public readonly clickhouseErrorType?: string; + constructor(message: string, context: { query: string }, clickhouseErrorType?: string) { super({ message, context, }); + this.clickhouseErrorType = clickhouseErrorType; } } + +/** + * ClickHouse error types raised when a query exceeds a server-side resource limit + * (`max_execution_time`, `max_memory_usage`, etc.). These mean the caller's query was too + * expensive, not a service fault, so callers can turn them into an actionable 4xx. + */ +const CLICKHOUSE_RESOURCE_LIMIT_ERROR_TYPES = new Set([ + "MEMORY_LIMIT_EXCEEDED", + "TIMEOUT_EXCEEDED", + "TOO_SLOW", + "TOO_MANY_ROWS", + "TOO_MANY_BYTES", + "TOO_MANY_ROWS_OR_BYTES", +]); + +export function isClickhouseResourceLimitError(error: unknown): boolean { + return ( + error instanceof QueryError && + error.clickhouseErrorType !== undefined && + CLICKHOUSE_RESOURCE_LIMIT_ERROR_TYPES.has(error.clickhouseErrorType) + ); +} diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 407c33135cc..100101a5635 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -75,6 +75,7 @@ import { } from "./errors.js"; export { msToClickHouseInterval } from "./intervals.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; +import type { Meter } from "@internal/tracing"; import type { Agent as HttpAgent } from "http"; import type { Agent as HttpsAgent } from "https"; @@ -123,7 +124,7 @@ export { export type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql"; // Errors -export { QueryError } from "./client/errors.js"; +export { QueryError, isClickhouseResourceLimitError } from "./client/errors.js"; export type ClickhouseCommonConfig = { keepAlive?: { @@ -133,6 +134,7 @@ export type ClickhouseCommonConfig = { httpAgent?: HttpAgent | HttpsAgent; clickhouseSettings?: ClickHouseSettings; logger?: Logger; + meter?: Meter; logLevel?: LogLevel; compression?: { request?: boolean; @@ -178,6 +180,7 @@ export class ClickHouse { url: config.url, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, @@ -195,6 +198,7 @@ export class ClickHouse { url: config.readerUrl, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, @@ -207,6 +211,7 @@ export class ClickHouse { url: config.writerUrl, clickhouseSettings: config.clickhouseSettings, logger: this.logger, + meter: config.meter, logLevel: config.logLevel, keepAlive: config.keepAlive, httpAgent: config.httpAgent, From ee2939386288e6994345a2325b4d5bef4f1b42c2 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 15:54:22 +0200 Subject: [PATCH 14/28] perf(webapp): cache deployment logs across navigations (#4775) Switching between deployments in the dashboard re-fetched the whole build log stream from record zero and re-rendered the list line by line every time. Logs are now cached per deployment for the lifetime of the tab: revisiting a deployment shows its logs immediately, and the stream is resumed from the next unread record rather than restarted. Finished deployments whose stream has been read through the `finalized` event are served entirely from the cache. ### Changes The stream/cache logic moved out of the route into a `useDeploymentLogs` hook. On each deployment switch it seeds state from the cache, resumes the S2 read session at `nextSeqNum`, and writes back on cleanup or natural session end. Completion is derived from the stream's own `finalized` event (plus a terminal deployment status), not from the session closing, so a session cut short by token expiry or a proxy cannot pin a truncated log in the cache. Memory is bounded by a small LRU (`deploymentLogsCache`): at most 20 deployments and 20,000 log lines in total, least recently viewed evicted first. The most recently viewed deployment is always kept, so a single very large log can temporarily exceed the line budget on its own. Records are batched into one state update per tick instead of one per line. --- .../runs/v3/deploymentLogsCache.test.ts | 97 ++++++++++ .../components/runs/v3/deploymentLogsCache.ts | 57 ++++++ apps/webapp/app/hooks/useDeploymentLogs.ts | 169 ++++++++++++++++++ .../route.tsx | 120 +------------ 4 files changed, 330 insertions(+), 113 deletions(-) create mode 100644 apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts create mode 100644 apps/webapp/app/components/runs/v3/deploymentLogsCache.ts create mode 100644 apps/webapp/app/hooks/useDeploymentLogs.ts diff --git a/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts b/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts new file mode 100644 index 00000000000..3253b11411b --- /dev/null +++ b/apps/webapp/app/components/runs/v3/deploymentLogsCache.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { DeploymentLogsCache, type DeploymentLogEntry } from "./deploymentLogsCache"; + +function lines(count: number): DeploymentLogEntry[] { + return Array.from({ length: count }, (_, i) => ({ + message: `line ${i}`, + timestamp: new Date(0), + level: "info" as const, + })); +} + +describe("DeploymentLogsCache", () => { + it("returns undefined for unknown keys", () => { + const cache = new DeploymentLogsCache(2, 100); + expect(cache.get("missing")).toBeUndefined(); + }); + + it("stores and returns entries", () => { + const cache = new DeploymentLogsCache(2, 100); + const value = { logs: lines(3), nextSeqNum: 3, finalized: true, complete: true }; + cache.set("a", value); + expect(cache.get("a")).toBe(value); + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(3); + }); + + it("evicts the least recently used deployment past the entry limit", () => { + const cache = new DeploymentLogsCache(2, 100); + cache.set("a", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("b", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.get("a"); + cache.set("c", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.size).toBe(2); + }); + + it("evicts oldest deployments past the total line budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("b", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("c", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.lineCount).toBe(8); + }); + + it("always keeps the entry just set, even when it alone exceeds the budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("big", { logs: lines(50), nextSeqNum: 50, finalized: true, complete: true }); + + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("big")?.logs).toHaveLength(50); + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(50); + }); + + it("treats replacing a key as a recent use", () => { + const cache = new DeploymentLogsCache(2, 100); + cache.set("a", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("b", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + cache.set("a", { logs: lines(2), nextSeqNum: 2, finalized: true, complete: true }); + cache.set("c", { logs: lines(1), nextSeqNum: 1, finalized: false, complete: false }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")?.logs).toHaveLength(2); + expect(cache.get("c")).toBeDefined(); + }); + + it("keeps recently read deployments when evicting for the line budget", () => { + const cache = new DeploymentLogsCache(10, 10); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.set("b", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + cache.get("a"); + cache.set("c", { logs: lines(4), nextSeqNum: 4, finalized: true, complete: true }); + + expect(cache.get("b")).toBeUndefined(); + expect(cache.get("a")).toBeDefined(); + expect(cache.get("c")).toBeDefined(); + expect(cache.lineCount).toBe(8); + }); + + it("replaces an existing key without double counting lines", () => { + const cache = new DeploymentLogsCache(10, 100); + cache.set("a", { logs: lines(4), nextSeqNum: 4, finalized: false, complete: false }); + cache.set("a", { logs: lines(6), nextSeqNum: 6, finalized: true, complete: true }); + + expect(cache.size).toBe(1); + expect(cache.lineCount).toBe(6); + expect(cache.get("a")?.complete).toBe(true); + }); +}); diff --git a/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts b/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts new file mode 100644 index 00000000000..0cb3977437d --- /dev/null +++ b/apps/webapp/app/components/runs/v3/deploymentLogsCache.ts @@ -0,0 +1,57 @@ +export type DeploymentLogEntry = { + message: string; + timestamp: Date; + level: "info" | "error" | "warn" | "debug"; +}; + +export type CachedDeploymentLogs = { + logs: readonly DeploymentLogEntry[]; + nextSeqNum: number; + finalized: boolean; + complete: boolean; +}; + +export class DeploymentLogsCache { + private entries = new Map(); + private totalLines = 0; + + constructor( + private readonly maxDeployments: number, + private readonly maxTotalLines: number + ) {} + + get(key: string): CachedDeploymentLogs | undefined { + const entry = this.entries.get(key); + if (!entry) return undefined; + this.entries.delete(key); + this.entries.set(key, entry); + return entry; + } + + set(key: string, value: CachedDeploymentLogs) { + const existing = this.entries.get(key); + if (existing) { + this.totalLines -= existing.logs.length; + this.entries.delete(key); + } + this.entries.set(key, value); + this.totalLines += value.logs.length; + + for (const [oldestKey, oldest] of this.entries) { + if (oldestKey === key) break; + if (this.entries.size <= this.maxDeployments && this.totalLines <= this.maxTotalLines) break; + this.entries.delete(oldestKey); + this.totalLines -= oldest.logs.length; + } + } + + get size() { + return this.entries.size; + } + + get lineCount() { + return this.totalLines; + } +} + +export const deploymentLogsCache = new DeploymentLogsCache(20, 20_000); diff --git a/apps/webapp/app/hooks/useDeploymentLogs.ts b/apps/webapp/app/hooks/useDeploymentLogs.ts new file mode 100644 index 00000000000..fb581aa87ea --- /dev/null +++ b/apps/webapp/app/hooks/useDeploymentLogs.ts @@ -0,0 +1,169 @@ +import { S2, S2Error } from "@s2-dev/streamstore"; +import { DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; +import type { WorkerDeploymentStatus } from "@trigger.dev/database"; +import { useEffect, useState } from "react"; +import { + deploymentLogsCache, + type DeploymentLogEntry, +} from "~/components/runs/v3/deploymentLogsCache"; + +type DeploymentEventStream = { + s2: { + basin: string; + stream: string; + accessToken: string; + }; +}; + +const FINISHED_DEPLOYMENT_STATUSES = new Set([ + "DEPLOYED", + "FAILED", + "CANCELED", + "TIMED_OUT", +]); + +type UseDeploymentLogsOptions = { + eventStream: DeploymentEventStream | undefined; + status: WorkerDeploymentStatus; +}; + +export function useDeploymentLogs({ eventStream, status }: UseDeploymentLogsOptions) { + const [logs, setLogs] = useState([]); + const [isStreaming, setIsStreaming] = useState(true); + const [streamError, setStreamError] = useState(null); + + const basin = eventStream?.s2.basin; + const stream = eventStream?.s2.stream; + const accessToken = eventStream?.s2.accessToken; + + useEffect(() => { + if (!basin || !stream || !accessToken) return; + + const isFinished = FINISHED_DEPLOYMENT_STATUSES.has(status); + const cacheKey = `${basin}/${stream}`; + const cached = deploymentLogsCache.get(cacheKey); + + let entries = cached?.logs ?? []; + let nextSeqNum = cached?.nextSeqNum ?? 0; + let pending: DeploymentLogEntry[] = []; + let flushTimer: ReturnType | undefined; + let finalized = cached?.finalized ?? false; + + // oxlint-disable-next-line react/set-state-in-effect -- Seed from the cache when the selected deployment changes. + setLogs(entries); + setStreamError(null); + + if (cached?.complete) { + setIsStreaming(false); + return; + } + + setIsStreaming(true); + + const abortController = new AbortController(); + + const flush = () => { + clearTimeout(flushTimer); + flushTimer = undefined; + if (abortController.signal.aborted || pending.length === 0) return; + entries = entries.concat(pending); + pending = []; + setLogs(entries); + }; + + const push = (entry: DeploymentLogEntry) => { + pending.push(entry); + flushTimer ??= setTimeout(flush, 0); + }; + + const store = () => { + clearTimeout(flushTimer); + flushTimer = undefined; + if (pending.length > 0) { + entries = entries.concat(pending); + pending = []; + } + if (entries.length === 0 && nextSeqNum === 0 && !finalized) return; + deploymentLogsCache.set(cacheKey, { + logs: entries, + nextSeqNum, + finalized, + complete: finalized && isFinished, + }); + }; + + const streamLogs = async () => { + try { + const s2Stream = new S2({ accessToken }).basin(basin).stream(stream); + + do { + const readSession = await s2Stream.readSession( + { + start: { from: { seqNum: nextSeqNum }, clamp: true }, + stop: { waitSecs: 60 }, + }, + { signal: abortController.signal } + ); + + for await (const record of readSession) { + nextSeqNum = record.seqNum + 1; + + const decoded = record.body; + const result = DeploymentEventFromString.safeParse(decoded); + + if (!result.success) { + // fallback to the previous format in s2 logs for compatibility + const headers: Record = {}; + if (record.headers) { + for (const [name, value] of record.headers) { + headers[name] = value; + } + } + const level = + (headers["level"]?.toLowerCase() as DeploymentLogEntry["level"]) ?? "info"; + + push({ timestamp: new Date(record.timestamp), message: decoded, level }); + continue; + } + + const event = result.data; + if (event.type === "finalized") finalized = true; + if (event.type !== "log") continue; + + push({ + timestamp: new Date(record.timestamp), + message: event.data.message, + level: event.data.level, + }); + } + } while (!abortController.signal.aborted && !finalized && !isFinished); + } catch (error) { + if (abortController.signal.aborted) return; + + if (error instanceof S2Error && error.code === "stream_not_found") { + finalized = isFinished; + return; + } + if (error instanceof S2Error && error.code === "permission_denied") return; + + console.error("Failed to stream logs:", error); + setStreamError("Failed to stream logs"); + } finally { + if (!abortController.signal.aborted) { + flush(); + setIsStreaming(false); + store(); + } + } + }; + + streamLogs(); + + return () => { + abortController.abort(); + store(); + }; + }, [basin, stream, accessToken, status]); + + return { logs, isStreaming, streamError }; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx index 343df5bff82..bfa7e5918fb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx @@ -2,7 +2,6 @@ import { useLocation } from "@remix-run/react"; import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { useEffect, useState, useRef, useCallback } from "react"; -import { S2, S2Error } from "@s2-dev/streamstore"; import { Clipboard, ClipboardCheck, @@ -51,7 +50,8 @@ import { cn } from "~/utils/cn"; import { v3DeploymentParams, v3DeploymentsPath, v3RunsPath } from "~/utils/pathBuilder"; import { capitalizeWord } from "~/utils/string"; import { UserTag } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments/route"; -import { DeploymentEventFromString } from "@trigger.dev/core/v3/schemas"; +import { useDeploymentLogs } from "~/hooks/useDeploymentLogs"; +import { type DeploymentLogEntry } from "~/components/runs/v3/deploymentLogsCache"; import { deploymentAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; import type { Handle } from "~/utils/handle"; @@ -91,12 +91,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } }; -type LogEntry = { - message: string; - timestamp: Date; - level: "info" | "error" | "warn" | "debug"; -}; - function getTriggeredViaDisplay(triggeredVia: string | null | undefined): { icon: React.ReactNode; label: string; @@ -205,110 +199,10 @@ export default function Page() { const page = new URLSearchParams(location.search).get("page"); const logsDisabled = eventStream === undefined; - const [logs, setLogs] = useState([]); - const [isStreaming, setIsStreaming] = useState(true); - const [streamError, setStreamError] = useState(null); - const isPending = deployment.status === "PENDING"; - - useEffect(() => { - if (logsDisabled) return; - - const abortController = new AbortController(); - - // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. - setLogs([]); - setStreamError(null); - setIsStreaming(true); - - const streamLogs = async () => { - try { - const s2 = new S2({ accessToken: eventStream.s2.accessToken }); - const basin = s2.basin(eventStream.s2.basin); - const stream = basin.stream(eventStream.s2.stream); - - const readSession = await stream.readSession( - { - start: { from: { seqNum: 0 }, clamp: true }, - stop: { waitSecs: 60 }, - }, - { signal: abortController.signal } - ); - - for await (const record of readSession) { - const decoded = record.body; - const result = DeploymentEventFromString.safeParse(decoded); - - if (!result.success) { - // fallback to the previous format in s2 logs for compatibility - try { - const headers: Record = {}; - - if (record.headers) { - for (const [name, value] of record.headers) { - headers[name] = value; - } - } - const level = (headers["level"]?.toLowerCase() as LogEntry["level"]) ?? "info"; - - setLogs((prevLogs) => [ - ...prevLogs, - { - timestamp: new Date(record.timestamp), - message: decoded, - level, - }, - ]); - } catch (err) { - console.error("Failed to parse log record:", err); - } - - continue; - } - - const event = result.data; - if (event.type !== "log") { - continue; - } - - setLogs((prevLogs) => [ - ...prevLogs, - { - timestamp: new Date(record.timestamp), - message: event.data.message, - level: event.data.level, - }, - ]); - } - } catch (error) { - if (abortController.signal.aborted) return; - - const isNotFoundError = - error instanceof S2Error && - error.code && - ["permission_denied", "stream_not_found"].includes(error.code); - if (isNotFoundError) return; - - console.error("Failed to stream logs:", error); - setStreamError("Failed to stream logs"); - } finally { - if (!abortController.signal.aborted) { - setIsStreaming(false); - } - } - }; - - streamLogs(); - - return () => { - abortController.abort(); - }; - }, [ - eventStream?.s2?.basin, - eventStream?.s2?.stream, - eventStream?.s2?.accessToken, - isPending, - logsDisabled, - ]); + const { logs, isStreaming, streamError } = useDeploymentLogs({ + eventStream, + status: deployment.status, + }); return (
@@ -622,7 +516,7 @@ function LogsDisplay({ streamError, initialCollapsed = false, }: { - logs: LogEntry[]; + logs: readonly DeploymentLogEntry[]; isStreaming: boolean; streamError: string | null; initialCollapsed?: boolean; From 97d70b890696929c1afd1e7700d0f7e2ff0503fc Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:30:14 +0100 Subject: [PATCH 15/28] feat(run-store): make the run-ops router correct at N shards (#4771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Makes `RoutingRunStore` correct when the run-ops layer routes across more than two Postgres stores. Today it routes between a gen-1 `new` dedicated database and a `legacy` control-plane database; this generalizes every routing policy to N shards while keeping the two-store behaviour byte-identical. The change sets the four routing decisions that were implicit in code order, and fixes one hazard that failed silently: - **Id → shard key.** The router resolves a shard key with `resolveShard` instead of the binary residency classifier, so a gen-2 id reaches its own shard through the keyed map. - **Membership vs routing.** `#distinctStores` (one entry per physical database, aliases excluded by a declared `aliasOf`) drives every sum, probe, and merge; `#shards` drives routing. An aliased shard can no longer make a sum count one database twice. - **Probe order.** A keyless lookup stays a sequential short-circuit at two stores; above two it fans out in parallel, picks by precedence, tolerates a single down leg, and keeps the canonical not-found throw on the legacy leg. - **Precedence and duplicates.** One merge helper across all four merge sites. A duplicate id confined to `{new, legacy}` stays silent (the known drain-mirror case); any other cross-shard duplicate increments `runops_shard_duplicate_id_total` and logs at error level. - **Disjoint sum (the silent hazard).** `countPendingWaitpoints` and the waitpoint collector now partition absent ids by shard and **union by id** rather than summing counts. A drain-mirrored waitpoint on both gen-1 stores is counted once, so a blocked run can no longer hang forever on a double-counted pending waitpoint. - **Waitpoint completion.** A gen-2 waitpoint completes on its own shard, overriding the legacy pins; a cuid waitpoint keeps its two-member gen-1-pair probe unchanged. - **Fail-loud creates.** A create with no shard key throws instead of silently defaulting to `new`. An id resolving to an unconfigured shard throws instead of being dropped. Two new counters are exported: `runops_shard_duplicate_id_total` and `runops_waitpoint_probe_fallback_total`. ## Why it is safe to merge With only `{new, legacy}` configured every generalized rule reduces to today's behaviour. `resolveShard` returns exactly what the old classifier returned for every id shape that exists today, and no gen-2 id is minted yet. The only intentional behaviour change is the fail-loud create throw; an enumeration of production call sites confirmed no caller trips it. ## Testing - New container-free algebra suite (50 cases) over probe order, precedence, the duplicate alarm, the disjoint-sum partition, the waitpoint probes, and the fail-loud paths. - New `runOpsStore.nShardMatrix.test.ts` runs a four-store matrix (legacy + new + two gen-2 shards) against real Postgres containers: the disjoint-sum union, the alias topology, cross-tree completion, pagination merges, and mixed-id hydration. - New `makeNShardRunOpsPostgresTest(k)` fixture in `@internal/testcontainers`. - Full run-store corpus green: 71 files, 480 tests. Typecheck, lint, format, and knip all clean. ## Notes - Draft: opened for review; not marking ready yet. - No changeset or `.server-changes` file: internal routing infrastructure, no user-visible behaviour change. - TRI-13427. --- apps/webapp/app/v3/runStore.server.ts | 38 +- .../run-store/src/PostgresRunStore.ts | 2 +- internal-packages/run-store/src/index.ts | 1 + .../run-store/src/routingStoreMetrics.ts | 16 + .../src/runOpsStore.nShardMatrix.test.ts | 305 ++++++++ .../src/runOpsStore.runKeyedRouting.test.ts | 5 +- .../src/runOpsStore.shardMap.test.ts | 676 +++++++++++++++++- .../run-store/src/runOpsStore.ts | 630 ++++++++++++---- internal-packages/run-store/src/types.ts | 2 + internal-packages/testcontainers/src/index.ts | 87 +++ .../testcontainers/src/nShardFixture.test.ts | 27 + 11 files changed, 1622 insertions(+), 167 deletions(-) create mode 100644 internal-packages/run-store/src/routingStoreMetrics.ts create mode 100644 internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts create mode 100644 internal-packages/testcontainers/src/nShardFixture.test.ts diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 9ccf84b5117..15e860e65c3 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -1,5 +1,12 @@ -import { PostgresRunStore, RoutingRunStore, type RunStore } from "@internal/run-store"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { + PostgresRunStore, + RoutingRunStore, + type RoutingStoreMetrics, + type RunStore, +} from "@internal/run-store"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { Counter } from "prom-client"; +import { metricsRegister } from "~/metrics.server"; import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { @@ -29,8 +36,8 @@ type BuildRunStoreDeps = { /** Single-DB store handles (control-plane pair). Used verbatim when split is OFF. */ singleWriter: PrismaClient; singleReplica: PrismaReplicaClient; - /** Residency classifier; defaults to ownerEngine inside RoutingRunStore. */ - classify?: (id: string) => Residency; + /** Id-to-shard-key resolver; defaults to the core resolveShard inside RoutingRunStore. */ + resolveShard?: (id: string) => ShardKey; /** Per-pool transaction-resilience configs threaded into the store(s) this builds (IoC). */ singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; @@ -82,10 +89,31 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { return new RoutingRunStore({ new: newStore, legacy: legacyStore, - classify: deps.classify ?? ownerEngine, + resolveShard: deps.resolveShard ?? resolveShard, + metrics: routingStoreMetrics, }); } +// singleton: module-scope Counter registration double-registers under dev HMR. +const routingStoreMetrics: RoutingStoreMetrics = singleton("routingStoreMetrics", () => { + const duplicateId = new Counter({ + name: "runops_shard_duplicate_id_total", + help: "One id was returned by two run-ops shards that must be disjoint (a routing-invariant violation).", + labelNames: ["shard_keys"], + registers: [metricsRegister], + }); + const probeFallback = new Counter({ + name: "runops_waitpoint_probe_fallback_total", + help: "A waitpoint was not on the run-ops store its id named and was found by a fallback probe.", + labelNames: ["from", "to"], + registers: [metricsRegister], + }); + return { + recordDuplicateId: (shardKeys) => duplicateId.inc({ shard_keys: shardKeys.join(",") }), + recordWaitpointProbeFallback: (from, to) => probeFallback.inc({ from, to }), + }; +}); + // Build the routing store whenever BOTH run-ops DBs are configured, independent of // RUN_OPS_SPLIT_ENABLED. Reads must fan out across both DBs so a run that lives on the new // DB stays visible even with the flag off (matches the db.server topology factory). The flag diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index b7d5086431b..33604d01148 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -1826,7 +1826,7 @@ export class PostgresRunStore implements RunStore { const branches = args.idempotencyKeys.map((key) => { const base = params.length; params.push(args.runtimeEnvironmentId, args.taskIdentifier, key); - return `SELECT "friendlyId", "idempotencyKey", "idempotencyKeyExpiresAt" FROM "TaskRun" WHERE "runtimeEnvironmentId" = $${base + 1} AND "taskIdentifier" = $${base + 2} AND "idempotencyKey" = $${base + 3}`; + return `SELECT "id", "createdAt", "friendlyId", "idempotencyKey", "idempotencyKeyExpiresAt" FROM "TaskRun" WHERE "runtimeEnvironmentId" = $${base + 1} AND "taskIdentifier" = $${base + 2} AND "idempotencyKey" = $${base + 3}`; }); return prisma.$queryRawUnsafe( branches.join(" UNION ALL "), diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 8893975cf12..a4109ab0104 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,4 +3,5 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./routingStoreMetrics.js"; export * from "./snapshotComparator.js"; diff --git a/internal-packages/run-store/src/routingStoreMetrics.ts b/internal-packages/run-store/src/routingStoreMetrics.ts new file mode 100644 index 00000000000..d4aa454d903 --- /dev/null +++ b/internal-packages/run-store/src/routingStoreMetrics.ts @@ -0,0 +1,16 @@ +/** + * Counters the routing store emits. Injected the same way RedisSnapshotStore takes its metrics, + * so the package stays free of a metrics dependency and a test can assert on a fake. + * + * runops_shard_duplicate_id_total — one id returned by two shards that should be disjoint + * runops_waitpoint_probe_fallback_total — a waitpoint was not on the store its id named + */ +export type RoutingStoreMetrics = { + recordDuplicateId(shardKeys: string[]): void; + recordWaitpointProbeFallback(from: string, to: string): void; +}; + +export const noopRoutingStoreMetrics: RoutingStoreMetrics = { + recordDuplicateId() {}, + recordWaitpointProbeFallback() {}, +}; diff --git a/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts b/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts new file mode 100644 index 00000000000..3f42e36171a --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.nShardMatrix.test.ts @@ -0,0 +1,305 @@ +// FOUR-STORE MATRIX — proves RoutingRunStore is correct across legacy + new + two gen-2 shards +// (a, b) against REAL databases (makeNShardRunOpsPostgresTest). NEVER mocked. This is where the +// §3.4 disjoint-sum fix is proven end-to-end: a double count here strands a blocked run forever. +// +// runOpsStore.mixedResidency.test.ts is the TWO-store invariant lock and stays byte-identical; this +// file is the N-store extension and lives separately. + +import { makeNShardRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { resolveShard } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const matrixTest = makeNShardRunOpsPostgresTest(2); + +// A gen-2 id: 24-char base32hex core, the shard char at index 24, version "2" at index 25. +// resolveShard(gen2("a", ...)) === "a". A bare cuid-length id classifies "legacy". +function gen2(shardChar: string, seed: string): string { + const core = (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24); + return `${core}${shardChar}2`; +} +function cuid(seed: string): string { + return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY +} + +function makeStore(prisma: AnyClient, variant: "legacy" | "dedicated") { + return new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant: variant, + }); +} + +// The real four-store split: legacy (full schema) + new + gen-2 a + gen-2 b (dedicated subset), +// routed by the REAL core resolveShard. +function makeMatrixRouter( + legacyPrisma: PrismaClient, + newPrisma: RunOpsPrismaClient, + shardPrismas: RunOpsPrismaClient[] +) { + return new RoutingRunStore({ + new: makeStore(newPrisma, "dedicated"), + legacy: makeStore(legacyPrisma, "legacy"), + shards: [ + { key: "a", store: makeStore(shardPrismas[0]!, "dedicated") }, + { key: "b", store: makeStore(shardPrismas[1]!, "dedicated") }, + ], + resolveShard, + }); +} + +async function seedLegacyEnv(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + environmentId: environment.id, + }; +} + +function buildRun(params: { + runId: string; + runtimeEnvironmentId: string; + organizationId: string; + projectId: string; + createdAt?: Date; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${params.runId}`, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: params.createdAt ?? new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +async function seedPendingWaitpoint( + prisma: AnyClient, + params: { id: string; projectId: string; environmentId: string } +) { + await (prisma as PrismaClient).waitpoint.create({ + data: { + id: params.id, + friendlyId: `wp_${params.id}`, + type: "MANUAL", + status: "PENDING", + idempotencyKey: `idem_${params.id}`, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + }); +} + +describe("RoutingRunStore four-store matrix — disjoint sum on real databases", () => { + matrixTest( + "countPendingWaitpoints unions across a gen-2 shard and the gen-1 pair with no double count", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "disjoint"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + + // The blocked run lives on shard a. + const runId = gen2("a", "run"); + // Its blocking waitpoints: a gen-2 waitpoint on shard b, a cuid on legacy, and a cuid MIRRORED + // onto both gen-1 stores (the drain-mirror case that must count once). + const wpB = gen2("b", "wpb"); + const wpCuid = cuid("wpcuid"); + const wpMirror = cuid("wpmirror"); + + const dedicatedEnv = { projectId: env.projectId, environmentId: env.environmentId }; + await seedPendingWaitpoint(shardPrismas[1]!, { id: wpB, ...dedicatedEnv }); + await seedPendingWaitpoint(legacyPrisma, { id: wpCuid, ...dedicatedEnv }); + await seedPendingWaitpoint(legacyPrisma, { id: wpMirror, ...dedicatedEnv }); + await seedPendingWaitpoint(newPrisma, { id: wpMirror, ...dedicatedEnv }); + + // b:wpB (1) + legacy:wpCuid (1) + wpMirror (once, though on both gen-1 stores) = 3. + const count = await router.countPendingWaitpoints([wpB, wpCuid, wpMirror], undefined, runId); + expect(count).toBe(3); + } + ); + + matrixTest( + "a gen-2 waitpoint on the run's own shard contributes exactly once, not twice", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "ownshard"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + const runId = gen2("a", "run2"); + const wpA = gen2("a", "wpa"); + await seedPendingWaitpoint(shardPrismas[0]!, { + id: wpA, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // wpA lives on the run's own shard a → found by the presence query, never re-queried elsewhere. + expect(await router.countPendingWaitpoints([wpA], undefined, runId)).toBe(1); + } + ); +}); + +describe("RoutingRunStore four-store matrix — alias topology", () => { + matrixTest( + "an aliased gen-2 shard counts its database ONCE in a sum (declaration, not identity)", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "alias"); + // Shard "a" aliases "new" over the SAME database, but via a SEPARATE store object built over + // the same client — exactly how the wiring layer will construct it. Identity dedupe would see + // two objects and double-count; declaration dedupe counts the database once. + const newStore = makeStore(newPrisma, "dedicated"); + const aStoreSameDb = makeStore(newPrisma, "dedicated"); // distinct object, same DB + const router = new RoutingRunStore({ + new: newStore, + legacy: makeStore(legacyPrisma, "legacy"), + shards: [{ key: "a", store: aStoreSameDb, aliasOf: "new" }], + resolveShard, + }); + + const wp = cuid("aliaswp"); + await seedPendingWaitpoint(newPrisma, { + id: wp, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // No runId → the id-less sum fans over DISTINCT databases. The aliased "a" must not add a + // second leg over the "new" database, or the one pending waitpoint counts twice. + expect(await router.countPendingWaitpoints([wp])).toBe(1); + } + ); +}); + +describe("RoutingRunStore four-store matrix — mixed gen-1 and gen-2 reads", () => { + matrixTest( + "findRunsByIds hydrates a mixed id set across legacy, new and both gen-2 shards", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "mixed"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + + const legacyId = cuid("mixleg"); + // A v1 run-ops id (version "1") routes to "new"; gen-2 ids route to their shard char. + const newId = ("mixnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; + const aId = gen2("a", "mixa"); + const bId = gen2("b", "mixb"); + const all = [legacyId, newId, aId, bId]; + + for (const runId of all) { + await router.createRun(buildRun({ runId, ...env })); + } + + const found = await router.findRunsByIds(all, { select: { id: true } }); + expect(new Set([...found.keys()])).toEqual(new Set(all)); + } + ); +}); + +describe("RoutingRunStore four-store matrix — cross-tree completion across gen-2 shards", () => { + matrixTest( + "a gen-2 waitpoint completes on its own shard even under the cross-tree legacy pin", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "crosstree"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + // The waitpoint is owned by a run on shard b; the blocked run is on shard a (cross-tree). + const wpB = gen2("b", "ctwp"); + await seedPendingWaitpoint(shardPrismas[1]!, { + id: wpB, + projectId: env.projectId, + environmentId: env.environmentId, + }); + // isCrossTreeIdempotency pins gen-1 flows to legacy; a gen-2 id must OVERRIDE that pin, or the + // completion write lands on legacy, matches zero rows, and strands the run. + const store = await router.forWaitpointCompletion(wpB, { + isCrossTreeIdempotency: true, + } as never); + // The returned store finds wpB on its primary — only shard b holds it, so the override worked. + const found = await store.findWaitpoint({ where: { id: wpB } }, store.primaryReadClient); + expect(found?.id).toBe(wpB); + } + ); +}); + +describe("RoutingRunStore four-store matrix — pagination merge", () => { + matrixTest( + "findRuns merges an open-predicate page across all four stores in orderBy order", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const env = await seedLegacyEnv(legacyPrisma, "paginate"); + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); + // One run per store, distinct createdAt so the global sort order is unambiguous. + const rows = [ + { id: cuid("pgleg"), at: new Date("2024-01-01T00:00:00Z") }, + { + id: ("pgnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01", + at: new Date("2024-01-02T00:00:00Z"), + }, + { id: gen2("a", "pga"), at: new Date("2024-01-03T00:00:00Z") }, + { id: gen2("b", "pgb"), at: new Date("2024-01-04T00:00:00Z") }, + ]; + for (const r of rows) { + await router.createRun(buildRun({ runId: r.id, ...env, createdAt: r.at })); + } + // Open predicate (no id set) → fan out + merge; take 2 skip 1 over createdAt desc. + const page = (await router.findRuns({ + where: { runtimeEnvironmentId: env.runtimeEnvironmentId }, + select: { id: true, createdAt: true }, + orderBy: { createdAt: "desc" }, + take: 2, + skip: 1, + })) as Array<{ id: string }>; + // Global desc order is b, a, new, legacy; skip 1 take 2 → [a, new]. + expect(page.map((r) => r.id)).toEqual([rows[2]!.id, rows[1]!.id]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts b/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts index e29ecb11cb8..e5cb229ef06 100644 --- a/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts +++ b/internal-packages/run-store/src/runOpsStore.runKeyedRouting.test.ts @@ -319,8 +319,9 @@ describe("RoutingRunStore.countPendingWaitpoints — route by runId then partiti "legacy_run" ); expect(count).toBe(1); - // Fallback queried the other store with ONLY the id missing on the run's store. - expect(newStore.calls.map((c) => c.method)).toEqual(["countPendingWaitpoints"]); + // Fallback queried the other store with ONLY the id missing on the run's store. It uses the + // presence variant so the results can be unioned by id (a drain mirror counts once at N). + expect(newStore.calls.map((c) => c.method)).toEqual(["countPendingWaitpointsWithPresence"]); expect(newStore.calls[0]?.args[0]).toEqual(["waitpoint_crosstree"]); }); diff --git a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts index c25574e7561..b6ea71e9f60 100644 --- a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts +++ b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts @@ -10,7 +10,7 @@ import type { ReadClient, RunStore } from "./types.js"; // MUST NOT assert invocation order for a PARALLEL fan-out: both legs are issued before either // resolves, so the order they are created in is not a behaviour. -type Slot = "new" | "legacy"; +type Slot = string; type Call = { slot: Slot; method: string }; @@ -19,8 +19,14 @@ type FakeConfig = { runs?: Array>; // Edge rows this store returns from findManyTaskRunWaitpoints, regardless of filter. edges?: Array>; + // Batch row this store returns from findBatchTaskRunById, regardless of filter. + batch?: Record | null; // Waitpoint rows this store returns from findWaitpoint, regardless of filter. waitpoint?: Record | null; + // Rows this store returns from findRunsByIdempotencyKeys, regardless of filter. + idempotencyMatches?: Array>; + // Waitpoint ids this store reports as pending (present = pending here) for count/collect probes. + pendingWaitpointIds?: string[]; }; type FakeStore = RunStore & { @@ -79,10 +85,42 @@ function fakeStore(slot: Slot, log: Call[], config: FakeConfig = {}): FakeStore return Promise.resolve({ slot } as never); }) as FakeStore["updateWaitpoint"], + createWaitpoint: ((_args: unknown) => { + record("createWaitpoint"); + return Promise.resolve({ slot } as never); + }) as FakeStore["createWaitpoint"], + + runInTransaction: ((_runId: unknown, fn: (store: unknown, tx: unknown) => unknown) => { + record("runInTransaction"); + return Promise.resolve(fn(store, {})); + }) as FakeStore["runInTransaction"], + findManyTaskRunWaitpoints: ((_args: unknown, _client?: ReadClient) => { record("findManyTaskRunWaitpoints"); return Promise.resolve((config.edges ?? []) as never); }) as FakeStore["findManyTaskRunWaitpoints"], + + updateManyWaitpoints: ((_args: unknown) => { + record("updateManyWaitpoints"); + return Promise.resolve({ count: 1 } as never); + }) as FakeStore["updateManyWaitpoints"], + + findRunsByIdempotencyKeys: ((_args: unknown, _client?: ReadClient) => { + record("findRunsByIdempotencyKeys"); + return Promise.resolve((config.idempotencyMatches ?? []) as never); + }) as FakeStore["findRunsByIdempotencyKeys"], + + findBatchTaskRunById: ((_id: unknown, _args?: unknown, _client?: ReadClient) => { + record("findBatchTaskRunById"); + return Promise.resolve((config.batch ?? null) as never); + }) as FakeStore["findBatchTaskRunById"], + + countPendingWaitpointsWithPresence: ((waitpointIds: string[], _client?: ReadClient) => { + record("countPendingWaitpointsWithPresence"); + const pending = new Set(config.pendingWaitpointIds ?? []); + const found = waitpointIds.filter((id) => pending.has(id)); + return Promise.resolve({ pendingIds: found, presentIds: found } as never); + }) as FakeStore["countPendingWaitpointsWithPresence"], }; return store as unknown as FakeStore; @@ -103,6 +141,12 @@ function buildRouter(newConfig: FakeConfig = {}, legacyConfig: FakeConfig = {}) const trace = (log: Call[]) => log.map((c) => `${c.slot}:${c.method}`); +// A parallel fan-out issues every leg before any resolves, so the ORDER legs appear in the log is +// NOT a behaviour and MUST NOT be asserted. Compare the multiset of slots instead. Order assertions +// via `trace` are valid only for the sequential (two-or-fewer-store) probe. +const slots = (log: Call[], method?: string) => + (method ? log.filter((c) => c.method === method) : log).map((c) => c.slot).sort(); + describe("RoutingRunStore #probeOrder — new then legacy, sequential", () => { it("probes new BEFORE legacy for an unrouted findRun", async () => { const { router, log } = buildRouter(); @@ -166,10 +210,19 @@ describe("RoutingRunStore id-less fallbacks — the two defaults differ by role" expect(trace(log)).toEqual(["new:createRun"]); }); - it("routes an id-less checkpoint create to new (#idlessRouteShard)", async () => { + it("throws for an id-less checkpoint create rather than defaulting to new", async () => { const { router, log } = buildRouter(); - await router.createTaskRunCheckpoint({ data: {} } as never); - expect(trace(log)).toEqual(["new:createTaskRunCheckpoint"]); + await expect(router.createTaskRunCheckpoint({ data: {} } as never)).rejects.toThrow( + "createTaskRunCheckpoint requires ownerRunId to route" + ); + expect(trace(log)).toEqual([]); + }); + + it("throws for a batch create with no id rather than defaulting to new", async () => { + const { router } = buildRouter(); + await expect(router.createBatchTaskRun({} as never)).rejects.toThrow( + "createBatchTaskRun requires data.id to route" + ); }); it("routes an id-less waitpoint update to legacy (#idlessWaitpointShard)", async () => { @@ -178,3 +231,618 @@ describe("RoutingRunStore id-less fallbacks — the two defaults differ by role" expect(trace(log)).toEqual(["legacy:updateWaitpoint"]); }); }); + +describe("RoutingRunStore id-to-shard-key seam", () => { + it("defaults to the core resolveShard when neither seam is injected", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + // A gen-1 v1 body (version "1" at index 25) routes to new. + await router.findRun({ id: "a".repeat(24) + "01" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("keeps the legacy classify seam working, so the corpus stays green", async () => { + const { router, log } = buildRouter(); + await router.findRun({ id: "new_run_1" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("prefers an injected resolveShard over classify", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + classify: () => "NEW", + resolveShard: () => "legacy", + }); + await router.findRun({ id: "anything" }); + expect(trace(log)).toEqual(["legacy:findRun"]); + }); + + // An id naming a shard nobody configured must fail loud rather than fall back to a default + // store, which would be a silent read against the wrong database. The throw is SYNCHRONOUS: + // routing happens before any query is issued, and `await store.findRun(...)` propagates it + // identically. Only a `.catch()`-style caller would see the difference. + it("throws for an id resolving to an unconfigured shard key", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + resolveShard: () => "a", + }); + expect(() => router.findRun({ id: "anything" })).toThrow( + 'no store is configured for shard key "a"' + ); + expect(trace(log)).toEqual([]); + }); +}); + +function buildNShardRouter(shardKeys: string[], opts: { aliasOf?: Record } = {}) { + const log: Call[] = []; + const newStore = fakeStore("new", log); + const legacyStore = fakeStore("legacy", log); + const byKey: Record = { new: newStore, legacy: legacyStore }; + const shards = shardKeys.map((key) => { + const aliasOf = opts.aliasOf?.[key]; + const store = aliasOf ? byKey[aliasOf]! : fakeStore(key as Slot, log); + byKey[key] = store; + return aliasOf ? { key, store, aliasOf } : { key, store }; + }); + const router = new RoutingRunStore({ + new: newStore, + legacy: legacyStore, + shards, + resolveShard: (id: string) => id.split(":")[0]!, + }); + return { router, log, byKey }; +} + +describe("RoutingRunStore #distinctStores — one entry per database", () => { + it("routes an id to its gen-2 shard", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.findRun({ id: "a:run_1" }); + expect(trace(log)).toEqual(["a:findRun"]); + }); + + it("counts an aliased shard's database ONCE in a sum", async () => { + // "a" aliases "new": two keys, one database. + const { router, log } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + const result = await router.updateManyWaitpoints({ + where: { status: "PENDING" }, + data: {}, + } as never); + expect(trace(log)).toEqual(["new:updateManyWaitpoints", "legacy:updateManyWaitpoints"]); + expect(result.count).toBe(2); + }); + + it("still routes an id whose key is an alias", async () => { + const { router, log, byKey } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + expect(byKey.a).toBe(byKey.new); + await router.findRun({ id: "a:run_1" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("keeps #fanOutPartitioned key-driven so an aliased bucket is not dropped", async () => { + const { router, log } = buildNShardRouter(["a"], { aliasOf: { a: "new" } }); + await router.findRunsByIds(["a:r1", "legacy:r2"]); + // Both buckets get a leg. The aliased bucket routes onto the shared store. + expect(log.filter((c) => c.method === "findRuns")).toHaveLength(2); + }); + + it("rejects an aliasOf naming an unconfigured key", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a" as Slot, log), aliasOf: "nope" }], + }) + ).toThrow('aliasOf "nope"'); + }); + + it("rejects a shard key that reuses a reserved key (new/legacy)", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "new", store: fakeStore("shadow" as Slot, log) }], + }) + ).toThrow("must be unique"); + }); + + it("rejects a duplicate custom shard key", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a1" as Slot, log) }, + { key: "a", store: fakeStore("a2" as Slot, log) }, + ], + }) + ).toThrow("must be unique"); + }); + + it("rejects a self-alias (a -> a), which would drop its database from every fan-out", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a" as Slot, log), aliasOf: "a" }], + }) + ).toThrow("must name a non-aliased store"); + }); + + it("rejects an alias chain (a -> b where b is itself aliased)", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a" as Slot, log), aliasOf: "b" }, + { key: "b", store: fakeStore("b" as Slot, log), aliasOf: "new" }, + ], + }) + ).toThrow("must name a non-aliased store"); + }); + + it("rejects an alias cycle (a -> b, b -> a), which would drop both databases", () => { + const log: Call[] = []; + expect( + () => + new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a" as Slot, log), aliasOf: "b" }, + { key: "b", store: fakeStore("b" as Slot, log), aliasOf: "a" }, + ], + }) + ).toThrow("must name a non-aliased store"); + }); +}); + +describe("RoutingRunStore probe at N", () => { + it("keeps the sequential short circuit at two distinct stores", async () => { + const { router, log } = buildRouter({ runs: [{ id: "r1" }] }); + await router.findRun({ spanId: "span_x" }); + expect(trace(log)).toEqual(["new:findRun"]); + }); + + it("issues every leg in parallel above two distinct stores", async () => { + // "b" carries the only hit. A sequential probe would stop the moment it found a result; a + // true parallel fan-out queries every OTHER leg too, since all legs are issued before any + // resolves. Miss-path (every leg misses) throw semantics are covered separately below. + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log) }, + { key: "b", store: fakeStore("b", log, { runs: [{ id: "r1" }] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await router.findRun({ spanId: "span_x" }); + expect(slots(log)).toEqual(["a", "b", "legacy", "new"]); + }); + + it("gives the legacy leg the canonical throw when every leg misses", async () => { + const { router } = buildNShardRouter(["a"]); + await expect(router.findRunOrThrow({ spanId: "span_x" })).rejects.toThrow("no run on legacy"); + }); + + it("tolerates a failing leg when another leg wins", async () => { + const log: Call[] = []; + const broken = fakeStore("a", log); + (broken as { findRun: unknown }).findRun = () => Promise.reject(new Error("shard a is down")); + const router = new RoutingRunStore({ + new: fakeStore("new", log, { runs: [{ id: "r1" }] }), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: broken }], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await expect(router.findRun({ spanId: "span_x" })).resolves.toMatchObject({ id: "r1" }); + }); + + it("surfaces a leg failure when no leg wins", async () => { + const log: Call[] = []; + const broken = fakeStore("a", log); + (broken as { findRun: unknown }).findRun = () => Promise.reject(new Error("shard a is down")); + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: broken }], + resolveShard: (id: string) => id.split(":")[0]!, + }); + await expect(router.findRun({ spanId: "span_x" })).rejects.toThrow("shard a is down"); + }); +}); + +describe("RoutingRunStore merge precedence and duplicate alarm", () => { + const spy = () => { + const seen: string[][] = []; + return { + metrics: { + recordDuplicateId: (k: string[]) => seen.push(k), + recordWaitpointProbeFallback() {}, + }, + seen, + }; + }; + + it("stays silent for a duplicate run id across the gen-1 pair", async () => { + const { metrics, seen } = spy(); + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { runs: [{ id: "dup", from: "new" }] }), + legacy: fakeStore("legacy", log, { runs: [{ id: "dup", from: "legacy" }] }), + metrics, + }); + const rows = (await router.findRuns({ + where: { runtimeEnvironmentId: "env_1" }, + select: { id: true, from: true }, + })) as Array<{ from: string }>; + expect(rows).toHaveLength(1); + expect(rows[0]?.from).toBe("new"); // NEW wins the precedence merge + expect(seen).toEqual([]); + }); + + it("alarms for a duplicate involving a gen-2 shard and still picks deterministically", async () => { + const { metrics, seen } = spy(); + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log, { runs: [{ id: "dup", from: "a" }] }) }, + { key: "b", store: fakeStore("b", log, { runs: [{ id: "dup", from: "b" }] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + metrics, + }); + const rows = (await router.findRuns({ + where: { runtimeEnvironmentId: "env_1" }, + select: { id: true, from: true }, + })) as Array<{ from: string }>; + expect(rows).toHaveLength(1); + expect(rows[0]?.from).toBe("b"); // last in #precedence [legacy, new, a, b] wins + expect(seen).toEqual([["a", "b"]]); + }); + + it("passes through an edge row whose projection omits id", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { edges: [{ taskRunId: "r1" }] }), + legacy: fakeStore("legacy", log, { edges: [{ taskRunId: "r2" }] }), + }); + const edges = (await router.findManyTaskRunWaitpoints({ + where: { waitpointId: "w" }, + select: { taskRunId: true }, + })) as Array<{ taskRunId: string }>; + expect(edges).toHaveLength(2); + }); +}); + +describe("RoutingRunStore findRunsByIdempotencyKeys tiebreak", () => { + const older = new Date("2026-01-01T00:00:00Z"); + const newer = new Date("2026-01-02T00:00:00Z"); + const match = (id: string, createdAt: Date) => ({ + id, + createdAt, + friendlyId: `run_${id}`, + idempotencyKey: "k", + idempotencyKeyExpiresAt: null, + }); + + it("keeps NEW-wins across the gen-1 pair even when legacy is older", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { idempotencyMatches: [match("n1", newer)] }), + legacy: fakeStore("legacy", log, { idempotencyMatches: [match("l1", older)] }), + }); + const rows = await router.findRunsByIdempotencyKeys({ + runtimeEnvironmentId: "env", + taskIdentifier: "t", + idempotencyKeys: ["k"], + }); + expect(rows.map((r) => r.id)).toEqual(["n1"]); + }); + + it("takes the earliest createdAt once a gen-2 shard is involved", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { idempotencyMatches: [match("n1", newer)] }), + legacy: fakeStore("legacy", log), + shards: [ + { key: "a", store: fakeStore("a", log, { idempotencyMatches: [match("a1", older)] }) }, + ], + resolveShard: (id: string) => id.split(":")[0]!, + }); + const rows = await router.findRunsByIdempotencyKeys({ + runtimeEnvironmentId: "env", + taskIdentifier: "t", + idempotencyKeys: ["k"], + }); + expect(rows.map((r) => r.id)).toEqual(["a1"]); + }); +}); + +describe("RoutingRunStore countPendingWaitpoints — disjoint-sum partition", () => { + // resolveShard: "x:..." -> "x" (a gen-2 shard); a bare cuid -> "legacy". + function partitionRouter(pending: Record, spy?: (k: string[]) => void) { + const log: Call[] = []; + const mk = (slot: string) => fakeStore(slot, log, { pendingWaitpointIds: pending[slot] ?? [] }); + const router = new RoutingRunStore({ + new: mk("new"), + legacy: mk("legacy"), + shards: [ + { key: "a", store: mk("a") }, + { key: "b", store: mk("b") }, + ], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + ...(spy ? { metrics: { recordDuplicateId: spy, recordWaitpointProbeFallback() {} } } : {}), + }); + return { router, log }; + } + const countCalls = (log: Call[]) => slots(log, "countPendingWaitpointsWithPresence"); + + it("sends a gen-2 absent id to its own shard only", async () => { + const { router, log } = partitionRouter({ b: ["b:w1"] }); + expect(await router.countPendingWaitpoints(["b:w1"], undefined, "a:run")).toBe(1); + expect(countCalls(log)).toEqual(["a", "b"]); // run shard a (presence) + fallback b + }); + + it("contributes zero for a gen-2 id whose shard IS the run's shard", async () => { + const { router, log } = partitionRouter({}); + expect(await router.countPendingWaitpoints(["a:w1"], undefined, "a:run")).toBe(0); + expect(countCalls(log)).toEqual(["a"]); // absent on a; no fallback leg (b-bucket empty, a skipped) + }); + + it("probes BOTH gen-1 stores for a cuid absent id when the run is on a gen-2 shard", async () => { + const { router, log } = partitionRouter({ legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "a:run")).toBe(1); + expect(countCalls(log)).toEqual(["a", "legacy", "new"]); + }); + + it("probes only legacy for a cuid when the run is on new", async () => { + const { router, log } = partitionRouter({ legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "new:run")).toBe(1); + // run shard resolves "new:run" -> "new" (presence); cuid -> {legacy, new} minus new = legacy + expect(countCalls(log)).toEqual(["legacy", "new"]); + }); + + it("probes only new for a cuid when the run is on legacy", async () => { + const { router, log } = partitionRouter({ new: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "cuid_run")).toBe(1); + // run shard resolves cuid -> "legacy"; cuid -> {legacy, new} minus legacy = new only + expect(countCalls(log)).toEqual(["legacy", "new"]); + }); + + it("counts a drain-mirrored cuid ONCE and stays silent", async () => { + const seen: string[][] = []; + const { router } = partitionRouter({ new: ["cuid_w1"], legacy: ["cuid_w1"] }, (k) => + seen.push(k) + ); + expect(await router.countPendingWaitpoints(["cuid_w1"], undefined, "a:run")).toBe(1); + expect(seen).toEqual([]); // the gen-1 mirror is expected, never alarmed + }); + + it("fails loud when an absent id resolves to an unconfigured shard key", async () => { + // "c:w1" resolves to shard "c", which is not configured. Silently dropping it would under-count + // a pending waitpoint and prematurely unblock the run — so it must throw, not skip. + const { router } = partitionRouter({}); + await expect(router.countPendingWaitpoints(["c:w1"], undefined, "a:run")).rejects.toThrow( + 'unconfigured shard key "c"' + ); + }); + + it("returns zero for an id absent everywhere", async () => { + const { router } = partitionRouter({}); + expect(await router.countPendingWaitpoints(["b:w9"], undefined, "a:run")).toBe(0); + }); + + it("id-less count unions a drain-mirrored cuid to one, never sums it to two", async () => { + // No runId: fans over distinct stores. A cuid pending on BOTH gen-1 stores is one waitpoint. + const { router } = partitionRouter({ new: ["cuid_w1"], legacy: ["cuid_w1"] }); + expect(await router.countPendingWaitpoints(["cuid_w1"])).toBe(1); + }); + + it("returns the true total for a mixed gen-2 and cuid set with no double count", async () => { + const { router } = partitionRouter({ + b: ["b:w1"], + legacy: ["cuid_w1"], + new: ["cuid_w1", "cuid_w2"], + }); + const count = await router.countPendingWaitpoints( + ["b:w1", "cuid_w1", "cuid_w2", "b:w9"], + undefined, + "a:run" + ); + expect(count).toBe(3); // b:w1 + cuid_w1 (mirror, once) + cuid_w2; b:w9 absent + }); +}); + +describe("RoutingRunStore waitpoint probes at N", () => { + it("routes a gen-2 waitpoint directly, with no probe", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.updateWaitpoint({ where: { id: "b:w1" }, data: {} } as never); + expect(trace(log)).toEqual(["b:updateWaitpoint"]); + }); + + it("keeps a cuid waitpoint on the gen-1 pair and never probes a gen-2 shard", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { waitpoint: { id: "cuid_w1" } }), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + await router.updateWaitpoint({ where: { id: "cuid_w1" }, data: {} } as never); + expect(log.map((c) => c.slot)).not.toContain("a"); + }); + + it("records a probe fallback when the waitpoint is not on the store its id names", async () => { + const falls: Array<[string, string]> = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log, { waitpoint: { id: "cuid_w1" } }), + legacy: fakeStore("legacy", log, { waitpoint: null }), + metrics: { + recordDuplicateId() {}, + recordWaitpointProbeFallback: (from, to) => falls.push([from, to]), + }, + }); + await router.updateWaitpoint({ where: { id: "cuid_w1" }, data: {} } as never); + expect(falls).toEqual([["legacy", "new"]]); + }); + + it("lets a gen-2 waitpoint id beat the cross-tree legacy pin", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + const store = await router.forWaitpointCompletion("b:w1", { + isCrossTreeIdempotency: true, + } as never); + expect((store as FakeStore).slot).toBe("b"); + expect(log.map((c) => c.slot)).not.toContain("legacy"); + }); + + it("keeps the legacy pin for a cuid waitpoint in a cross-tree completion", async () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { waitpoint: { id: "cuid_w1" } }), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + const store = await router.forWaitpointCompletion("cuid_w1", { + isCrossTreeIdempotency: true, + } as never); + expect((store as FakeStore).slot).toBe("legacy"); + expect(log.map((c) => c.slot)).not.toContain("a"); + }); +}); + +describe("RoutingRunStore gen-2 shard refuses a co-located cuid waitpoint", () => { + // resolveShard: "x:..." -> "x" (gen-2 shard); a bare id (no colon) -> "legacy". + function coLocateRouter() { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + shards: [{ key: "a", store: fakeStore("a", log) }], + resolveShard: (id: string) => (id.includes(":") ? id.split(":")[0]! : "legacy"), + }); + return { router, log }; + } + + it("throws when a cuid waitpoint is co-located onto a gen-2 shard", () => { + const { router, log } = coLocateRouter(); + // Routing throws SYNCHRONOUSLY, before any write is issued. + expect(() => + router.createWaitpoint({ data: { id: "cuid_w1" } } as never, undefined, { + coLocateWithRunId: "a:run_1", + }) + ).toThrow('onto gen-2 shard "a"'); + expect(trace(log)).toEqual([]); + }); + + it("throws when an id-less waitpoint is co-located onto a gen-2 shard", () => { + // Prisma's @default(cuid()) would otherwise mint a cuid on the gen-2 shard AFTER the write, + // leaving it unroutable for its own completion (CodeRabbit finding). Reject it up front. + const { router, log } = coLocateRouter(); + expect(() => + router.createWaitpoint({ data: {} } as never, undefined, { coLocateWithRunId: "a:run_1" }) + ).toThrow('onto gen-2 shard "a"'); + expect(trace(log)).toEqual([]); + }); + + it("allows a gen-2 waitpoint co-located onto its own shard", async () => { + const { router, log } = coLocateRouter(); + await router.createWaitpoint({ data: { id: "a:w1" } } as never, undefined, { + coLocateWithRunId: "a:run_1", + }); + expect(trace(log)).toEqual(["a:createWaitpoint"]); + }); + + it("allows a cuid waitpoint co-located onto a gen-1 store", async () => { + const { router, log } = coLocateRouter(); + await router.createWaitpoint({ data: { id: "cuid_w1" } } as never, undefined, { + coLocateWithRunId: "legacy_run", + }); + expect(trace(log)).toEqual(["legacy:createWaitpoint"]); + }); +}); + +describe("RoutingRunStore id-less read/route defaults hold at N (never a gen-2 shard)", () => { + // With gen-2 shards a and b configured, each id-less default must still resolve to its named + // gen-1 store, never leak to a gen-2 shard. + it("#routeOrNew falls back to new for an id-less create", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.createRun({ data: {} } as never); + expect(trace(log)).toEqual(["new:createRun"]); + }); + + it("#routeOrNew falls back to new for an id-less runInTransaction", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.runInTransaction(undefined, async () => undefined); + expect(trace(log)).toEqual(["new:runInTransaction"]); + }); + + it("#resolveWaitpointStore(undefined) falls back to legacy for an id-less update", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.updateWaitpoint({ where: { idempotencyKey: "k" }, data: {} } as never); + expect(trace(log)).toEqual(["legacy:updateWaitpoint"]); + }); + + it("#waitpointWriteStore with no owner and no residency falls back to legacy", async () => { + const { router, log } = buildNShardRouter(["a", "b"]); + await router.createWaitpoint({ data: {} } as never); + expect(trace(log)).toEqual(["legacy:createWaitpoint"]); + }); +}); + +describe("RoutingRunStore batch probe tolerates legitimate dual-residency", () => { + it("does NOT alarm when a batch is found on a gen-2 shard AND legacy", async () => { + const seen: string[][] = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { batch: { id: "batch_dup", from: "legacy" } }), + shards: [{ key: "a", store: fakeStore("a", log, { batch: { id: "batch_dup", from: "a" } }) }], + resolveShard: (id: string) => id.split(":")[0]!, + metrics: { recordDuplicateId: (k) => seen.push(k), recordWaitpointProbeFallback() {} }, + }); + const batch = (await router.findBatchTaskRunById("batch_dup")) as { from: string } | null; + // batchTriggerV3 writes raw to the control plane while runEngine routes by id, so this is a + // legitimate dual-residency, not a routing-invariant violation — no alarm. + expect(seen).toEqual([]); + // Precedence still picks deterministically (gen-2 shard 'a' outranks legacy). + expect(batch?.from).toBe("a"); + }); + + it("still alarms when a RUN is found on a gen-2 shard AND legacy (real violation)", async () => { + const seen: string[][] = []; + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log, { runs: [{ id: "dup", from: "legacy" }] }), + shards: [{ key: "a", store: fakeStore("a", log, { runs: [{ id: "dup", from: "a" }] }) }], + resolveShard: (id: string) => id.split(":")[0]!, + metrics: { recordDuplicateId: (k) => seen.push(k), recordWaitpointProbeFallback() {} }, + }); + await router.findRun({ spanId: "span_x" }); + expect(seen).toEqual([["legacy", "a"]]); + }); +}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 27bd78866d4..53089da21a5 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -7,7 +7,11 @@ import type { TaskRunStatus, WaitpointTag, } from "@trigger.dev/database"; -import { ownerEngine, type Residency, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { + resolveShard as coreResolveShard, + type Residency, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; import type { ClearIdempotencyKeyInput, @@ -30,8 +34,10 @@ import type { TaskRunWithWaitpoint, WaitpointColocationOptions, } from "./types.js"; +import { Logger } from "@trigger.dev/core/logger"; import { isReadReplicaClient } from "./readReplicaClient.js"; import { CONNECTED_RUNS_LIMIT } from "./PostgresRunStore.js"; +import { noopRoutingStoreMetrics, type RoutingStoreMetrics } from "./routingStoreMetrics.js"; import { boundedIn } from "@trigger.dev/database"; @@ -42,20 +48,19 @@ const LEGACY_SHARD: ShardKey = "legacy"; /** * Run-ops routing substrate for the TaskRun-core method group. Implements {@link RunStore} over a - * map from shard key to store, selecting one by the residency classifier (`ownerEngine`: run-ops - * id→NEW, cuid→LEGACY). The compat constructor holds the two gen-1 shards — a NEW store (the + * map from shard key to store, selecting one by `resolveShard` (gen-2 id→its shard char, gen-1 + * run-ops id→NEW, cuid→LEGACY). The compat constructor holds the two gen-1 shards — a NEW store (the * dedicated run-ops DB, where new runs are born) and a LEGACY store (the control-plane DB). * Inert until the injecting seam wires it in under `isSplitEnabled()`; reads no flag here. * - * Every shard MUST be a distinct database. Single-DB does not construct this class at all — the - * injecting seam returns a bare PostgresRunStore — and split mode requires two configured run-ops - * URLs whose distinctness the boot sentinel enforces fail-closed. Two shard keys that resolve to - * ONE store would make the sum sites (#sumCounts, the counting fan-outs) count that store twice. + * Two shard keys MAY resolve to one store only through a declared `aliasOf`. #distinctStores then + * holds one entry per database, so a sum never counts a database twice. An undeclared duplicate + * store is still a configuration error. * - * Three policies are held as data rather than implied by statement order: {@link #probeOrder} for a - * lookup with no routable id, {@link #precedence} for a merge, and the two id-less fallbacks. A - * merge MUST iterate #precedence and a probe MUST iterate #probeOrder — the two are the reverse of - * each other, so swapping them changes behaviour. + * #probeOrder and #precedence each list one key per distinct store. At two shards they are exact + * reverses. At N they are not: both put gen-2 shards last, so a probe finds the gen-1 pair first + * and a merge lets a gen-2 shard win. A probe MUST iterate #probeOrder and a merge MUST iterate + * #precedence. */ export class RoutingRunStore implements RunStore { readonly #shards: ReadonlyMap; @@ -65,25 +70,90 @@ export class RoutingRunStore implements RunStore { // Ascending authority for a merge. The last write wins, so the highest-authority shard wins a // duplicate id. Every merge in this class MUST use this order. readonly #precedence: readonly ShardKey[]; + // One entry per distinct database, in precedence order. A fan-out sum iterates this, never #shards. + readonly #distinctStores: ReadonlyArray<{ key: ShardKey; store: RunStore }>; // The two id-less defaults. They differ by role on purpose: a route with no id lands on the // steady-state home, a waitpoint read with no id lands on the legacy store. readonly #idlessRouteShard: ShardKey; readonly #idlessWaitpointShard: ShardKey; - readonly #classify: (id: string) => Residency; + // Id to shard key. `resolveShard` is the gen-2 seam. `classify` is the gen-1 seam, kept because + // five sites inject it and three of those are the regression corpus. A gen-1 classifier can + // only ever name the two reserved keys, so it can never reach a gen-2 shard. + readonly #resolveShardKey: (id: string) => ShardKey; + readonly #metrics: RoutingStoreMetrics; + readonly #logger: Logger; // Compat constructor: the two gen-1 stores, keyed by their reserved shard keys. The options type // MUST stay closed — a union arm loosens the excess-property check and retires the // `@ts-expect-error onLegacyRead` lock in the test corpus. - constructor(options: { new: RunStore; legacy: RunStore; classify?: (id: string) => Residency }) { + constructor(options: { + new: RunStore; + legacy: RunStore; + classify?: (id: string) => Residency; + resolveShard?: (id: string) => ShardKey; + shards?: ReadonlyArray<{ key: ShardKey; store: RunStore; aliasOf?: ShardKey }>; + metrics?: RoutingStoreMetrics; + logger?: Logger; + }) { + const shards = options.shards ?? []; + // Keys must be unique across the reserved pair and every configured shard. A key of "new" or + // "legacy" would overwrite the reserved #shards entry; a repeated custom key would appear twice + // in #precedence and #distinctStores, so a fan-out would query one database twice. + const shardKeys = [NEW_SHARD, LEGACY_SHARD, ...shards.map((s) => s.key)]; + if (new Set(shardKeys).size !== shardKeys.length) { + throw new Error( + "RoutingRunStore: shard keys must be unique and cannot reuse the reserved 'new' or 'legacy' keys" + ); + } + const configured = new Set(shardKeys); + const aliasedKeys = new Set(shards.filter((s) => s.aliasOf !== undefined).map((s) => s.key)); + for (const shard of shards) { + if (shard.aliasOf === undefined) { + continue; + } + // An alias must name a REAL root store, so #distinctStores keeps exactly one entry per + // database. A target that is itself aliased (a chain or a cycle) would drop every key in the + // cycle from #distinctStores, and that database would vanish from every read and write. + if (!configured.has(shard.aliasOf)) { + throw new Error( + `RoutingRunStore: shard "${shard.key}" declares aliasOf "${shard.aliasOf}", which is not configured` + ); + } + if (shard.aliasOf === shard.key || aliasedKeys.has(shard.aliasOf)) { + throw new Error( + `RoutingRunStore: shard "${shard.key}" aliasOf "${shard.aliasOf}" must name a non-aliased store; chains and cycles are not allowed` + ); + } + } + this.#shards = new Map([ [NEW_SHARD, options.new], [LEGACY_SHARD, options.legacy], + ...shards.map((s) => [s.key, s.store] as const), ]); - this.#probeOrder = [NEW_SHARD, LEGACY_SHARD]; - this.#precedence = [LEGACY_SHARD, NEW_SHARD]; + + const gen2Keys = shards.map((s) => s.key); + this.#probeOrder = [NEW_SHARD, LEGACY_SHARD, ...gen2Keys]; + this.#precedence = [LEGACY_SHARD, NEW_SHARD, ...gen2Keys]; this.#idlessRouteShard = NEW_SHARD; this.#idlessWaitpointShard = LEGACY_SHARD; - this.#classify = options.classify ?? ownerEngine; + const classify = options.classify; + this.#resolveShardKey = + options.resolveShard ?? + (classify !== undefined + ? (id: string) => (classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD) + : coreResolveShard); + + // One entry per physical database, in precedence order. A declared alias contributes none: + // it shares its target's database, and a second leg over one database double-counts a sum. + // The discriminator is the DECLARATION, not object identity — the wiring layer may build a + // second store object over a shared client. + this.#distinctStores = this.#precedence + .filter((key) => !aliasedKeys.has(key)) + .map((key) => ({ key, store: this.#shardStore(key) })); + + this.#metrics = options.metrics ?? noopRoutingStoreMetrics; + this.#logger = options.logger ?? new Logger("RoutingRunStore", "warn"); } // A routing store spans two databases and has no single primary — routed reads resolve the @@ -113,9 +183,99 @@ export class RoutingRunStore implements RunStore { return store; } - // The shard that owns an existing id. Throws only when an injected classifier throws. + // A duplicate id is EXPECTED across the gen-1 pair (drain mirrors a token onto both). Any other + // combination breaks id-determinism: alarm, but keep the deterministic pick. + #reportDuplicateId(id: string, shardKeys: ShardKey[]): void { + if (shardKeys.every((key) => key === NEW_SHARD || key === LEGACY_SHARD)) { + return; + } + this.#metrics.recordDuplicateId(shardKeys); + this.#logger.error("RoutingRunStore: one id returned by two shards", { id, shardKeys }); + } + + // Merge key-tagged legs, keeping one row per id. Legs MUST arrive in #precedence order, so the + // highest-authority copy is written last and wins. A winner keeps the POSITION of its first + // sighting: callers observe row order whenever `orderBy` is absent. Rows whose projection omits + // `id` cannot be deduped and pass through unchanged. A duplicate whose reporting keys leave the + // gen-1 pair alarms via #reportDuplicateId. + #mergeById>(legs: Array<{ key: ShardKey; rows: R[] }>): R[] { + const byId = new Map(); + const keysById = new Map(); + const passthrough: R[] = []; + for (const { key, rows } of legs) { + for (const row of rows) { + const id = row.id; + if (typeof id !== "string") { + passthrough.push(row); + continue; + } + byId.set(id, row); + const keys = keysById.get(id); + if (keys) keys.push(key); + else keysById.set(id, [key]); + } + } + for (const [id, keys] of keysById) { + if (keys.length > 1) this.#reportDuplicateId(id, keys); + } + return [...byId.values(), ...passthrough]; + } + + // Where to look for waitpoint ids the run's own shard did not return. A gen-2 id names exactly + // one shard, so it goes there and nowhere else — that is what keeps the legs disjoint and the sum + // sound. A cuid names no shard: drain can mirror it onto NEW while it keeps its id, and a gen-2 + // run can block on a pre-gen-2 cuid token, so there is no single gen-1 partner. Both gen-1 stores + // are probed and the results are de-duped by id. A target equal to `runKey` is skipped (already + // probed); an id resolving to an unconfigured shard fails loud rather than being dropped. + #partitionAbsentIds(runKey: ShardKey, ids: string[]): Array<{ key: ShardKey; ids: string[] }> { + const byKey = new Map(); + const push = (key: ShardKey, id: string) => { + // The run's own shard was already probed, so skip it. But an id resolving to a shard nobody + // configured is UnknownShardKey: silently dropping it here would UNDER-count a pending + // waitpoint and prematurely unblock the run — the exact failure this method guards against. + // Fail loud instead (§7 append-only rule). + if (key === runKey) return; + if (!this.#shards.has(key)) { + throw new Error( + `RoutingRunStore: waitpoint "${id}" resolves to unconfigured shard key "${key}"` + ); + } + const bucket = byKey.get(key); + if (bucket) bucket.push(id); + else byKey.set(key, [id]); + }; + for (const id of ids) { + const home = this.#shardKeyOfSafe(id); + if (home === LEGACY_SHARD) { + push(LEGACY_SHARD, id); + push(NEW_SHARD, id); + } else { + push(home, id); + } + } + return this.#precedence + .filter((key) => byKey.has(key)) + .map((key) => ({ key, ids: byKey.get(key)! })); + } + + // A cuid is deliberately probed on BOTH gen-1 stores, so the same id from both is the expected + // drain mirror, not a violation. Any other id maps to one shard, so a two-leg return is a bug. + #isGen1MirrorProbe(id: string): boolean { + return this.#shardKeyOfSafe(id) === LEGACY_SHARD; + } + + // The gen-1 pair members other than `key`. A cuid waitpoint can only ever be drain-relocated + // BETWEEN the two gen-1 stores, so its "where does it really live" probe stays confined here and + // never touches a gen-2 shard. + #gen1PairExcept(key: ShardKey): Array<{ key: ShardKey; store: RunStore }> { + return [NEW_SHARD, LEGACY_SHARD] + .filter((k) => k !== key) + .map((k) => ({ key: k, store: this.#shardStore(k) })); + } + + // The shard that owns an existing id. Throws only when an injected resolver throws. #shardKeyOf(id: string): ShardKey { - return this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD; + return this.#resolveShardKey(id); } // An unclassifiable id is treated as LEGACY (probe the control-plane DB rather than drop a @@ -128,32 +288,88 @@ export class RoutingRunStore implements RunStore { } } - // Sequential probe over #probeOrder, returning the first non-null result. `isLast` marks the leg - // that owns the canonical not-found throw, so a caller can swap in its throwing variant there. + // #distinctStores sorted by `order`. Shared by #probeFirst and #fanOut so every ordered walk + // over the distinct-store set goes through one sort. + #orderedLegs(order: readonly ShardKey[]): ReadonlyArray<{ key: ShardKey; store: RunStore }> { + const rank = new Map(order.map((key, i) => [key, i])); + return [...this.#distinctStores].sort( + (a, b) => (rank.get(a.key) ?? 0) - (rank.get(b.key) ?? 0) + ); + } + + // A lookup with no routable id. At two distinct stores this is the sequential short circuit, + // byte-identical to before: the second leg is never queried when the first answers. Above two, + // a sequential walk would cost N round trips, so every leg is issued in parallel and the winner + // comes from #precedence. `isLast` marks the leg that owns the canonical not-found throw. async #probeFirst( - fn: (store: RunStore, key: ShardKey, isLast: boolean) => Promise + fn: (store: RunStore, key: ShardKey, isLast: boolean) => Promise, + opts?: { alarmOnDuplicate?: boolean } ): Promise { - const last = this.#probeOrder.length - 1; - for (let i = 0; i < last; i++) { - const key = this.#probeOrder[i]!; - const found = await fn(this.#shardStore(key), key, false); - if (found != null) { - return found; + const legs = this.#orderedLegs(this.#probeOrder); + const lastIndex = legs.length - 1; + + if (legs.length <= 2) { + for (let i = 0; i < lastIndex; i++) { + const { store, key } = legs[i]!; + const found = await fn(store, key, false); + if (found != null) { + return found; + } } + const { store, key } = legs[lastIndex]!; + return fn(store, key, true); } - const key = this.#probeOrder[last]!; - return fn(this.#shardStore(key), key, true); + + // Parallel. Every leg takes the NON-throwing arm, so one leg cannot reject a lookup another + // leg answers. A rejection is held and only surfaces when nothing was found. + const settled = await Promise.allSettled(legs.map(({ store, key }) => fn(store, key, false))); + + const hits: Array<{ key: ShardKey; value: Awaited }> = []; + let firstRejection: unknown; + settled.forEach((outcome, i) => { + if (outcome.status === "rejected") { + firstRejection ??= outcome.reason; + return; + } + if (outcome.value != null) { + hits.push({ key: legs[i]!.key, value: outcome.value }); + } + }); + + // A row on two stores is a routing-invariant violation for entities that live on exactly one + // store (runs, waitpoints, attempts, snapshots). Batches are the exception: `batchTriggerV3` + // writes raw to the control plane while runEngine routes by id, so a batch is legitimately + // dual-resident. Those callers pass `alarmOnDuplicate: false` so a batch on legacy + a gen-2 + // shard is not mistaken for a violation. + if (hits.length > 1 && opts?.alarmOnDuplicate !== false) { + this.#reportDuplicateId( + String((hits[0]!.value as { id?: unknown })?.id ?? "unknown"), + hits.map((h) => h.key) + ); + } + if (hits.length > 0) { + // Highest authority wins: #precedence ascends, so take the last hit in that order. + const rank = new Map(this.#precedence.map((key, i) => [key, i])); + hits.sort((a, b) => (rank.get(a.key) ?? 0) - (rank.get(b.key) ?? 0)); + return hits[hits.length - 1]!.value; + } + if (firstRejection !== undefined) { + throw firstRejection; + } + // Nothing found anywhere. The LEGACY leg owns the canonical not-found throw, so give it the + // throwing arm. One extra query, on the miss path only. + const legacy = this.#shardStore(LEGACY_SHARD); + return fn(legacy, LEGACY_SHARD, true); } - // Run `fn` on every shard in parallel, returning the results in `order`. Pass #probeOrder where - // the result-array order is observable; pass #precedence where a duplicate id's winner decides - // the value. The two orders are the reverse of each other, so passing the wrong one is a - // behaviour change. + // Run `fn` on every DISTINCT store in parallel. `order` selects the ordering; membership is + // always one entry per database, so an aliased key never contributes a second leg. #fanOut( order: readonly ShardKey[], fn: (store: RunStore, key: ShardKey) => Promise ): Promise { - return Promise.all(order.map((key) => fn(this.#shardStore(key), key))); + const legs = this.#orderedLegs(order); + return Promise.all(legs.map(({ store, key }) => fn(store, key))); } // Apply `fn` to every shard and sum the counts. A sum is order-independent, so this takes no order. @@ -174,6 +390,11 @@ export class RoutingRunStore implements RunStore { const byShard = new Map(); for (const id of ids) { const key = this.#shardKeyOfSafe(id); + // An id resolving to a shard nobody configured is UnknownShardKey. Dropping it would silently + // omit a row from the hydrated set, so fail loud (§7 append-only rule). + if (!this.#shards.has(key)) { + throw new Error(`RoutingRunStore: id "${id}" resolves to unconfigured shard key "${key}"`); + } const bucket = byShard.get(key); if (bucket) bucket.push(id); else byShard.set(key, [id]); @@ -188,13 +409,11 @@ export class RoutingRunStore implements RunStore { return Promise.all(legs); } - // Every shard other than `key`, in probe order. With the compat constructor this yields exactly - // one entry, which is why each caller may take the first. At more than two shards a caller MUST - // fan out over all of them instead. + // Every distinct store other than `key`'s, in precedence order. With the compat constructor this + // yields exactly one entry, which is why each caller may take the first. At more than two shards + // a caller MUST fan out over all of them instead. #shardsExcept(key: ShardKey): Array<{ key: ShardKey; store: RunStore }> { - return this.#probeOrder - .filter((k) => k !== key) - .map((k) => ({ key: k, store: this.#shardStore(k) })); + return this.#distinctStores.filter((s) => s.key !== key); } // A `findRuns` caller bound to the given store (preserves `this`; the overload set isn't @@ -270,21 +489,27 @@ export class RoutingRunStore implements RunStore { if (typeof id !== "string") { return home; } + // A gen-2 waitpoint carries its shard in its id and its row lives there. No probe. + if (homeKey !== NEW_SHARD && homeKey !== LEGACY_SHARD) { + return home; + } if ( await home.findWaitpoint({ where: { id } }, onPrimary ? home.primaryReadClient : undefined) ) { return home; } - const [other] = this.#shardsExcept(homeKey); - if (other === undefined) { - return home; + for (const { key, store } of this.#gen1PairExcept(homeKey)) { + if ( + await store.findWaitpoint( + { where: { id } }, + onPrimary ? store.primaryReadClient : undefined + ) + ) { + this.#metrics.recordWaitpointProbeFallback(homeKey, key); + return store; + } } - return (await other.store.findWaitpoint( - { where: { id } }, - onPrimary ? other.store.primaryReadClient : undefined - )) - ? other.store - : home; + return home; } static #waitpointId(clause: unknown): string | undefined { @@ -468,12 +693,11 @@ export class RoutingRunStore implements RunStore { async #findRunsOpen(args: FindRunsArgs, client?: ReadClient): Promise { const { args: selArgs, addedFields } = ensureProjected(args); const fan = widenForMerge(selArgs); - const legs = await this.#fanOut(this.#precedence, (store) => - this.#findManyOn(store, client)(fan) - ); - const byId = new Map>(); - for (const r of legs.flat()) byId.set(r.id as string, r); - return finalizeRows([...byId.values()], args, addedFields); + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await this.#findManyOn(store, client)(fan)) as Record[], + })); + return finalizeRows(this.#mergeById(legs), args, addedFields); } // Canonical grouped replacement for `Promise.all(ids.map(id => readThroughRun(id)))`: reuses @@ -539,14 +763,45 @@ export class RoutingRunStore implements RunStore { if (args.idempotencyKeys.length === 0) { return []; } - const legs = await this.#fanOut(this.#precedence, (store) => - store.findRunsByIdempotencyKeys(args, RoutingRunStore.#ownPrimary(store, client)) + const legs = await this.#fanOut(this.#precedence, (store, key) => + store + .findRunsByIdempotencyKeys(args, RoutingRunStore.#ownPrimary(store, client)) + .then((rows) => ({ key, rows })) ); - const byKey = new Map(); - for (const row of legs.flat()) { - if (row.idempotencyKey != null) byKey.set(row.idempotencyKey, row); + // Dedupe by KEY, not id: two runs on two shards can legitimately share a global key. Across the + // gen-1 pair the winner stays today's precedence result (NEW wins), which the duplicate-guard + // contract depends on. Once a gen-2 shard supplies a candidate, order by creation instead, so + // the winner does not depend on configured shard order. + const byKey = new Map>(); + for (const { key, rows } of legs) { + for (const row of rows) { + if (row.idempotencyKey == null) continue; + const bucket = byKey.get(row.idempotencyKey); + if (bucket) bucket.push({ key, row }); + else byKey.set(row.idempotencyKey, [{ key, row }]); + } } - return [...byKey.values()]; + const out: IdempotencyKeyRunMatch[] = []; + for (const candidates of byKey.values()) { + const gen1Only = candidates.every(({ key }) => key === NEW_SHARD || key === LEGACY_SHARD); + if (gen1Only) { + out.push(candidates[candidates.length - 1]!.row); + continue; + } + out.push( + [...candidates].sort((a, b) => { + const byCreated = a.row.createdAt.getTime() - b.row.createdAt.getTime(); + return byCreated !== 0 + ? byCreated + : a.row.id < b.row.id + ? -1 + : a.row.id > b.row.id + ? 1 + : 0; + })[0]!.row + ); + } + return out; } // --------------------------------------------------------------------------- @@ -1153,10 +1408,20 @@ export class RoutingRunStore implements RunStore { runId?: string ): Promise { if (runId === undefined) { + // No run id to partition on: query every distinct store and UNION by id, matching the routed + // path below. A drain-mirrored cuid pending on both gen-1 stores must count once, not twice — + // summing raw counts here would reintroduce the double count this method exists to remove. const legs = await this.#fanOut(this.#probeOrder, (store) => - store.countPendingWaitpoints(waitpointIds, RoutingRunStore.#ownPrimary(store, client)) + store.countPendingWaitpointsWithPresence( + waitpointIds, + RoutingRunStore.#ownPrimary(store, client) + ) ); - return legs.reduce((sum, leg) => sum + leg, 0); + const union = new Set(); + for (const leg of legs) { + for (const id of leg.pendingIds) union.add(id); + } + return union.size; } if (waitpointIds.length === 0) { @@ -1173,15 +1438,41 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return pendingIds.length; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + const plan = this.#partitionAbsentIds(runKey, missing); + if (plan.length === 0) { return pendingIds.length; } - const otherPending = await other.store.countPendingWaitpoints( - missing, - RoutingRunStore.#ownPrimary(other.store, client) + const legs = await Promise.all( + plan.map(async ({ key, ids }) => { + const store = this.#shardStore(key); + const { pendingIds: found } = await store.countPendingWaitpointsWithPresence( + ids, + RoutingRunStore.#ownPrimary(store, client) + ); + return { key, found }; + }) ); - return pendingIds.length + otherPending; + // UNION by id, never a sum of counts. A cuid mirrored onto both gen-1 stores appears twice; + // summing it is exactly the double count that leaves pendingCount above zero forever and never + // unblocks the run. The run store's pending set seeds the union; missing ids are disjoint from + // it by construction. #reportDuplicateId is the tripwire for a NON-mirror two-leg return, which + // a consistent resolveShard makes unreachable — it guards a future partition bug. + const union = new Set(pendingIds); + const seenFrom = new Map(); + for (const { key, found } of legs) { + for (const id of found) { + const keys = seenFrom.get(id); + if (keys) keys.push(key); + else seenFrom.set(id, [key]); + union.add(id); + } + } + for (const [id, keys] of seenFrom) { + if (keys.length > 1 && !this.#isGen1MirrorProbe(id)) { + this.#reportDuplicateId(id, keys); + } + } + return union.size; } // Fan out and union: an id lives on exactly one store in steady state (a drain-mirror can put it on @@ -1238,7 +1529,19 @@ export class RoutingRunStore implements RunStore { waitpointId: string | undefined ): RunStore { if (ownerId !== undefined) { - return this.#shardStore(this.#shardKeyOfSafe(ownerId)); + const key = this.#shardKeyOfSafe(ownerId); + // A gen-2 shard holds only ids stamped for that shard, because a waitpoint completes on the + // shard its own id names. Anything else stranded the blocked run: a cuid (routes to the gen-1 + // pair on completion), an id for a DIFFERENT gen-2 shard, or NO id at all — Prisma's + // @default(cuid()) then mints a cuid on the gen-2 shard after the write. The mint layer must + // stamp the owner's shard onto the waitpoint id, so fail loud rather than write an orphan. + const isGen2 = key !== NEW_SHARD && key !== LEGACY_SHARD; + if (isGen2 && (waitpointId === undefined || this.#shardKeyOfSafe(waitpointId) !== key)) { + throw new Error( + `RoutingRunStore: refusing to co-locate waitpoint "${waitpointId ?? ""}" onto gen-2 shard "${key}"; its id must be stamped for that shard` + ); + } + return this.#shardStore(key); } if (residency !== undefined) { return this.#shardStore(residency === "NEW" ? NEW_SHARD : LEGACY_SHARD); @@ -1350,10 +1653,11 @@ export class RoutingRunStore implements RunStore { return rows; } - // Collect the scalar waitpoint rows (relation re-resolution happens in the caller). With a run id in - // scope and a bounded id set to partition on, route to the run's store and fall back to the other DB - // for ONLY the ids missing there (a rare cross-tree token) — the two legs are disjoint by - // construction, so no dedup is needed. Otherwise fan out to BOTH and dedup by id NEW-wins. + // Collect the scalar waitpoint rows (relation re-resolution happens in the caller). With a run id + // in scope and a bounded id set, route to the run's store and fall back for ONLY the ids missing + // there. The fallback targets come from #partitionAbsentIds: a gen-2 id to its own shard, a cuid + // to BOTH gen-1 stores. The cuid legs are not disjoint, so the fallback rows are merged by id. + // Otherwise (no bounded id set) fan out to every store and dedup by id NEW-wins. async #collectManyWaitpoints( scalarArgs: Record, client: ReadClient | undefined, @@ -1375,38 +1679,41 @@ export class RoutingRunStore implements RunStore { if (missing.length === 0) { return fromRun; } - const [other] = this.#shardsExcept(runKey); - if (other === undefined) { + // Same partition as countPendingWaitpoints: a gen-2 missing id goes to its own shard, a + // cuid to both gen-1 stores. The cuid legs are NOT disjoint, so merge by id (a drain mirror + // appears once) rather than concatenate. + const plan = this.#partitionAbsentIds(runKey, missing); + if (plan.length === 0) { return fromRun; } - const fromOther = (await other.store.findManyWaitpoints( - narrowArgsToIds(scalarArgs, missing) as Prisma.WaitpointFindManyArgs, - RoutingRunStore.#ownPrimary(other.store, client) - )) as Record[]; - return [...fromRun, ...fromOther]; + const legs = await Promise.all( + plan.map(async ({ key, ids }) => { + const store = this.#shardStore(key); + return { + key, + rows: (await store.findManyWaitpoints( + narrowArgsToIds(scalarArgs, ids) as Prisma.WaitpointFindManyArgs, + RoutingRunStore.#ownPrimary(store, client) + )) as Record[], + }; + }) + ); + return [...fromRun, ...this.#mergeById(legs)]; } // No bounded id set to partition on → fall through to the fan-out path. } - const legs = await this.#fanOut( - this.#precedence, - (store) => - store.findManyWaitpoints( - scalarArgs as Prisma.WaitpointFindManyArgs, - RoutingRunStore.#ownPrimary(store, client) - ) as Promise[]> - ); - // A token mirrored onto both DBs during drain appears in BOTH legs; dedup by id in #precedence - // order, so the highest-authority copy wins. Without this, edge-waitpoint hydration could read a - // stale LEGACY status and strand the run. Rows whose projection omits `id` pass through. - const byId = new Map>(); - const passthrough: Record[] = []; - for (const w of legs.flat()) { - const id = w.id; - if (typeof id === "string") byId.set(id, w); - else passthrough.push(w); - } - return [...byId.values(), ...passthrough]; + // A token mirrored onto both DBs during drain appears in BOTH legs; #mergeById dedups by id in + // #precedence order, so the highest-authority copy wins. Without this, edge-waitpoint hydration + // could read a stale LEGACY status and strand the run. + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyWaitpoints( + scalarArgs as Prisma.WaitpointFindManyArgs, + RoutingRunStore.#ownPrimary(store, client) + )) as Record[], + })); + return this.#mergeById(legs); } // Re-resolve a waitpoint's group-A relations across BOTH DBs and attach them to `row`. Each target @@ -1537,27 +1844,36 @@ export class RoutingRunStore implements RunStore { waitpointId: string, context: ForWaitpointCompletionContext ): Promise { - // Preferred store: explicit legacy-authority pins first, else the waitpoint's id-shape. + // A gen-2 waitpoint's row lives on the shard its id names. The three legacy pins encode "the one + // non-NEW store", a gen-1 idea, so a gen-2 id OVERRIDES them: honouring the pin would send the + // completion write to legacy, match zero rows, and strand the blocked run. A gen-2 id is also + // directly routable, so it takes no probe. + const idKey = this.#shardKeyOfSafe(waitpointId); + const isGen2 = idKey !== NEW_SHARD && idKey !== LEGACY_SHARD; + if (isGen2) { + return this.#shardStore(idKey); + } const preferredKey = context.treeOwnerResidency === "LEGACY" || context.isCrossTreeIdempotency === true || context.hasLegacyParent === true ? LEGACY_SHARD - : this.#shardKeyOfSafe(waitpointId); + : idKey; const preferred = this.#shardStore(preferredKey); - // Resolve to where the waitpoint ACTUALLY lives: a migrated run's waitpoint can be on NEW - // with a LEGACY-classified id (or vice versa), so verify and fall back rather than route - // by id-shape alone and miss it (which leaves the blocked run stuck forever). This guard - // selects the store a WRITE (updateManyWaitpoints) then lands on, so it must probe each - // store's PRIMARY (mirroring #resolveWaitpointStore's onPrimary): a just-created waitpoint the - // replica hasn't caught up on would otherwise mis-resolve the owner and strand the run. + // Resolve to where a CUID waitpoint ACTUALLY lives: a migrated run's waitpoint can be on NEW + // with a LEGACY-classified id (or vice versa), so verify and fall back across the gen-1 pair + // rather than route by id-shape alone and miss it (which leaves the blocked run stuck forever). + // This guard selects the store a WRITE (updateManyWaitpoints) then lands on, so it must probe + // each store's PRIMARY: a just-created waitpoint the replica has not caught up on would + // otherwise mis-resolve the owner and strand the run. if ( await preferred.findWaitpoint({ where: { id: waitpointId } }, preferred.primaryReadClient) ) { return preferred; } - for (const { store } of this.#shardsExcept(preferredKey)) { + for (const { key, store } of this.#gen1PairExcept(preferredKey)) { if (await store.findWaitpoint({ where: { id: waitpointId } }, store.primaryReadClient)) { + this.#metrics.recordWaitpointProbeFallback(preferredKey, key); return store; } } @@ -1596,13 +1912,14 @@ export class RoutingRunStore implements RunStore { RoutingRunStore.#ownPrimary(store, client) )) as Record[]; } else { - const legs = await this.#fanOut(this.#precedence, (store) => - store.findManyTaskRunWaitpoints( + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyTaskRunWaitpoints( scalarArgs as typeof args, RoutingRunStore.#ownPrimary(store, client) - ) - ); - edges = dedupeEdgesById(legs.flat()) as Record[]; + )) as Record[], + })); + edges = this.#mergeById(legs); } if (waitpoint) { @@ -1717,7 +2034,13 @@ export class RoutingRunStore implements RunStore { ownerRunId?: string, tx?: PrismaClientOrTransaction ): Promise> { - const store = this.#routeOrNew(ownerRunId); + // A create is a mint decision the mint layer owns. Defaulting to NEW was harmless with one + // dedicated store; at N it is a silent write to the wrong shard, and the run-routed snapshot's + // checkpointId FK then resolves on a different database. Fail loud instead. + if (ownerRunId === undefined) { + throw new Error("createTaskRunCheckpoint requires ownerRunId to route"); + } + const store = this.#route(ownerRunId); return store.createTaskRunCheckpoint(args, ownerRunId, undefined); } @@ -1731,9 +2054,12 @@ export class RoutingRunStore implements RunStore { ): Promise { // Route by the batch's classifiable internal id: run-ops id→NEW, cuid→LEGACY. The caller's // `tx` is never forwarded — the create runs on the owning store's own client so the batch and - // its co-resident child runs/items land on the same DB. Mirrors the by-id waitpoint-write routing / - // updateBatchTaskRun. - const store = await this.#routeOrNewForWrite(data.id); + // its co-resident child runs/items land on the same DB. A create with no id is a mint decision + // the mint layer must have made; at N a silent NEW default is a wrong-shard write, so fail loud. + if (data.id === undefined) { + throw new Error("createBatchTaskRun requires data.id to route"); + } + const store = await this.#routeForWrite(data.id); return store.createBatchTaskRun(data, undefined); } @@ -1763,8 +2089,9 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim (a cross-DB probe with one shared client can // only reach one DB); its presence resolves each leg to that store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunById(id, args, RoutingRunStore.#ownPrimary(store, client)) + return this.#probeFirst( + (store) => store.findBatchTaskRunById(id, args, RoutingRunStore.#ownPrimary(store, client)), + { alarmOnDuplicate: false } ); } @@ -1777,13 +2104,15 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim; its presence resolves each leg to that // store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunByFriendlyId( - friendlyId, - environmentId, - args, - RoutingRunStore.#ownPrimary(store, client) - ) + return this.#probeFirst( + (store) => + store.findBatchTaskRunByFriendlyId( + friendlyId, + environmentId, + args, + RoutingRunStore.#ownPrimary(store, client) + ), + { alarmOnDuplicate: false } ); } @@ -1802,13 +2131,15 @@ export class RoutingRunStore implements RunStore { ): Promise | null> { // Never forward the caller's client verbatim; its presence resolves each leg to that // store's OWN primary. - return this.#probeFirst((store) => - store.findBatchTaskRunByIdempotencyKey( - environmentId, - idempotencyKey, - args, - RoutingRunStore.#ownPrimary(store, client) - ) + return this.#probeFirst( + (store) => + store.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + RoutingRunStore.#ownPrimary(store, client) + ), + { alarmOnDuplicate: false } ); } @@ -1921,17 +2252,20 @@ export class RoutingRunStore implements RunStore { skip: 0, ...(args.take != null ? { take: skip + args.take } : {}), }; - const legs = await this.#fanOut(this.#precedence, (store) => - store.findManyWaitpointTags(perLeg, RoutingRunStore.#ownPrimary(store, client)) - ); - const byId = new Map(); - for (const tag of legs.flat()) byId.set(tag.id, tag); + const legs = await this.#fanOut(this.#precedence, async (store, key) => ({ + key, + rows: (await store.findManyWaitpointTags( + perLeg, + RoutingRunStore.#ownPrimary(store, client) + )) as unknown as Array>, + })); + const deduped = this.#mergeById(legs) as unknown as WaitpointTag[]; const merged = args.orderBy ? (sortByOrderBy( - [...byId.values()] as unknown as Array>, + deduped as unknown as Array>, args.orderBy as unknown as NonNullable ) as unknown as WaitpointTag[]) - : [...byId.values()]; + : deduped; return merged.slice(skip, args.take != null ? skip + args.take : undefined); } @@ -2055,20 +2389,6 @@ function narrowArgsToIds(args: Record, ids: string[]): Record(rows: R[]): R[] { - const byId = new Map(); - const passthrough: R[] = []; - for (const row of rows) { - const id = (row as { id?: unknown }).id; - if (typeof id === "string") byId.set(id, row); - else passthrough.push(row); - } - return [...byId.values(), ...passthrough]; -} - // A caller sub-select for an edge relation: `{ select?, include? }`, `true` for a bare `key: true`, // or undefined when not requested. type SubProjection = { select?: any; include?: any } | true | undefined; diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 7c7f9566893..3ceb0a462dd 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -23,6 +23,8 @@ import type { Residency } from "@trigger.dev/core/v3/isomorphic"; export type ReadClient = PrismaClientOrTransaction | PrismaReplicaClient; export type IdempotencyKeyRunMatch = { + id: string; + createdAt: Date; friendlyId: string; idempotencyKey: string | null; idempotencyKeyExpiresAt: Date | null; diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts index 8cdaee8571b..e72a63766e9 100644 --- a/internal-packages/testcontainers/src/index.ts +++ b/internal-packages/testcontainers/src/index.ts @@ -566,6 +566,93 @@ export const threeDbRunOpsPostgresTest = test.extend + test.extend({ + legacyUri: async ({}, use) => { + const container = await getWorkerPostgresContainer(); + const baseUri = container.getConnectionUri(); + const cloneDb = `nShardLegacy_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + try { + await use(postgresUriWithDatabase(baseUri, cloneDb)); + } finally { + await dropCloneDatabase(baseUri, cloneDb); + } + }, + newUri: async ({}, use) => { + const container = await getRunOpsWorkerPostgresContainer17(); + const baseUri = container.getConnectionUri(); + const cloneDb = `nShardNew_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + try { + await use(postgresUriWithDatabase(baseUri, cloneDb)); + } finally { + await dropCloneDatabase(baseUri, cloneDb); + } + }, + shardUris: async ({}, use) => { + const container = await getRunOpsWorkerPostgresContainer17(); + const baseUri = container.getConnectionUri(); + const clones: string[] = []; + try { + for (let i = 0; i < gen2ShardCount; i++) { + const cloneDb = `nShardGen2_${pgCloneCounter++}`; + await createDatabaseFromTemplate(baseUri, cloneDb); + clones.push(cloneDb); + } + await use(clones.map((db) => postgresUriWithDatabase(baseUri, db))); + } finally { + for (const db of clones) { + await dropCloneDatabase(baseUri, db); + } + } + }, + legacyPrisma: async ({ legacyUri }, use) => { + const prisma = new PrismaClient({ datasources: { db: { url: legacyUri } } }); + try { + await use(prisma); + } finally { + await prisma.$disconnect(); + } + }, + newPrisma: async ({ newUri }, use) => { + const prisma = new RunOpsPrismaClient({ datasources: { db: { url: newUri } } }); + try { + await use(prisma); + } finally { + await prisma.$disconnect(); + } + }, + shardPrismas: async ({ shardUris }, use) => { + const clients = shardUris.map( + (url) => new RunOpsPrismaClient({ datasources: { db: { url } } }) + ); + try { + await use(clients); + } finally { + for (const c of clients) { + await c.$disconnect(); + } + } + }, + }); + export const redisContainer = async ( { network, task }: { network: StartedNetwork } & TestContext, use: Use diff --git a/internal-packages/testcontainers/src/nShardFixture.test.ts b/internal-packages/testcontainers/src/nShardFixture.test.ts new file mode 100644 index 00000000000..1fd3449616f --- /dev/null +++ b/internal-packages/testcontainers/src/nShardFixture.test.ts @@ -0,0 +1,27 @@ +import { expect } from "vitest"; +import { makeNShardRunOpsPostgresTest } from "./index.js"; + +const nShardTest = makeNShardRunOpsPostgresTest(2); + +// Booting the PG14 + PG17 containers and cloning four databases on a cold runner far exceeds +// vitest's 5s default (this package sets no global testTimeout), so pass a generous per-test one. +nShardTest( + "builds 4 distinct databases (legacy + new + 2 gen-2 shards)", + async ({ legacyUri, newUri, shardUris }) => { + expect(shardUris).toHaveLength(2); + const all = [legacyUri, newUri, ...shardUris]; + expect(new Set(all).size).toBe(4); + }, + 120_000 +); + +nShardTest( + "each gen-2 clone carries the run-ops subset schema", + async ({ shardPrismas }) => { + expect(shardPrismas).toHaveLength(2); + for (const prisma of shardPrismas) { + await expect(prisma.taskRun.count()).resolves.toBe(0); + } + }, + 120_000 +); From 6a6f0a4960cb8bd3d3dd01301d36272738dfac77 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 25 Aug 2026 16:59:29 +0200 Subject: [PATCH 16/28] feat(webapp): pause deployment log auto-scroll on scroll-up (#4776) Auto-scroll now only follows while you are at the bottom. Scrolling up pauses it; scrolling back to the bottom, or clicking the new scroll-to-bottom button in the log header, resumes it. When you are at the bottom the same button scrolls to the top. Switching to another deployment starts at the bottom again. --- .../deployment-logs-follow-scroll.md | 6 ++ apps/webapp/app/hooks/useFollowScroll.ts | 100 ++++++++++++++++++ .../route.tsx | 33 ++++-- 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 .server-changes/deployment-logs-follow-scroll.md create mode 100644 apps/webapp/app/hooks/useFollowScroll.ts diff --git a/.server-changes/deployment-logs-follow-scroll.md b/.server-changes/deployment-logs-follow-scroll.md new file mode 100644 index 00000000000..d5fcf23ce8d --- /dev/null +++ b/.server-changes/deployment-logs-follow-scroll.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Deployment logs no longer jump to the bottom while you are reading earlier output. Scroll up to pause auto-scroll, and scroll back down or use the new scroll-to-bottom button in the log header to resume following. diff --git a/apps/webapp/app/hooks/useFollowScroll.ts b/apps/webapp/app/hooks/useFollowScroll.ts new file mode 100644 index 00000000000..4189f0377a7 --- /dev/null +++ b/apps/webapp/app/hooks/useFollowScroll.ts @@ -0,0 +1,100 @@ +import { useEffect, useLayoutEffect, useRef, useState, type RefObject } from "react"; + +const AT_BOTTOM_TOLERANCE_PX = 4; + +type FollowState = { + follow: boolean; + pinnedScrollTop: number | null; + lastScrollTop: number; + lastClientHeight: number; +}; + +export function useFollowScroll(containerRef: RefObject, content: unknown) { + const [isAtBottom, setIsAtBottom] = useState(true); + const stateRef = useRef({ + follow: true, + pinnedScrollTop: null, + lastScrollTop: 0, + lastClientHeight: 0, + }); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const state = stateRef.current; + state.lastScrollTop = container.scrollTop; + state.lastClientHeight = container.clientHeight; + + const onScroll = () => { + const { scrollTop, scrollHeight, clientHeight } = container; + const isEcho = scrollTop === state.pinnedScrollTop; + const movedUp = scrollTop < state.lastScrollTop && clientHeight <= state.lastClientHeight; + state.pinnedScrollTop = null; + state.lastScrollTop = scrollTop; + state.lastClientHeight = clientHeight; + if (isEcho) return; + + state.follow = !movedUp && scrollHeight - scrollTop - clientHeight <= AT_BOTTOM_TOLERANCE_PX; + setIsAtBottom(state.follow); + }; + + const onWheel = (event: WheelEvent) => { + if (event.deltaY >= 0 || !canScroll(container)) return; + state.follow = false; + setIsAtBottom(false); + }; + + container.addEventListener("scroll", onScroll, { passive: true }); + container.addEventListener("wheel", onWheel, { passive: true }); + return () => { + container.removeEventListener("scroll", onScroll); + container.removeEventListener("wheel", onWheel); + }; + }, [containerRef]); + + useLayoutEffect(() => { + const container = containerRef.current; + if (container && stateRef.current.follow) pinToBottom(container, stateRef.current); + }, [containerRef, isAtBottom, content]); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const observer = new ResizeObserver(() => { + if (stateRef.current.follow) pinToBottom(container, stateRef.current); + }); + observer.observe(container); + return () => observer.disconnect(); + }, [containerRef]); + + const scrollToBottom = () => { + const container = containerRef.current; + if (!container) return; + stateRef.current.follow = true; + setIsAtBottom(true); + pinToBottom(container, stateRef.current); + }; + + const scrollToTop = () => { + const container = containerRef.current; + if (!container || !canScroll(container)) return; + stateRef.current.follow = false; + setIsAtBottom(false); + container.scrollTop = 0; + }; + + return { isAtBottom, scrollToBottom, scrollToTop }; +} + +function pinToBottom(container: HTMLElement, state: FollowState) { + container.scrollTop = container.scrollHeight; + state.pinnedScrollTop = container.scrollTop; + state.lastScrollTop = container.scrollTop; + state.lastClientHeight = container.clientHeight; +} + +function canScroll(container: HTMLElement) { + return container.scrollHeight - container.clientHeight > AT_BOTTOM_TOLERANCE_PX; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx index bfa7e5918fb..a89752eba13 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx @@ -13,6 +13,8 @@ import { ServerIcon, } from "lucide-react"; import { ExitIcon } from "~/assets/icons/ExitIcon"; +import { MoveToBottomIcon } from "~/assets/icons/MoveToBottomIcon"; +import { MoveToTopIcon } from "~/assets/icons/MoveToTopIcon"; import { GitMetadata } from "~/components/GitMetadata"; import { VercelLink } from "~/components/integrations/VercelLink"; import { RuntimeIcon } from "~/components/RuntimeIcon"; @@ -51,6 +53,7 @@ import { v3DeploymentParams, v3DeploymentsPath, v3RunsPath } from "~/utils/pathB import { capitalizeWord } from "~/utils/string"; import { UserTag } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments/route"; import { useDeploymentLogs } from "~/hooks/useDeploymentLogs"; +import { useFollowScroll } from "~/hooks/useFollowScroll"; import { type DeploymentLogEntry } from "~/components/runs/v3/deploymentLogsCache"; import { deploymentAgentPageContext } from "~/components/dashboard-agent/suggested-prompts"; import type { Handle } from "~/utils/handle"; @@ -307,6 +310,7 @@ export default function Page() { Logs (null); + const { isAtBottom, scrollToBottom, scrollToTop } = useFollowScroll(logsContainerRef, logs); useEffect(() => { // oxlint-disable-next-line react/set-state-in-effect, react/no-deriving-state-in-effects -- Deployment status changes intentionally reset the user-controlled collapse state. setCollapsed(initialCollapsed); }, [initialCollapsed]); - // auto-scroll log container to bottom when new logs arrive - useEffect(() => { - if (logsContainerRef.current) { - logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; - } - }, [logs]); - const onCopyLogs = useCallback( (event: React.MouseEvent) => { event.preventDefault(); @@ -584,6 +582,27 @@ function LogsDisplay({
{logs.length > 0 && (
+ + + + {isAtBottom ? ( + + ) : ( + + )} + + + {isAtBottom ? "Scroll to top" : "Scroll to bottom"} + + + + Date: Tue, 25 Aug 2026 17:29:43 +0100 Subject: [PATCH 17/28] fix(webapp): disable browser autofill on environment variable inputs (#4777) The environment variable key and value inputs did not set an autocomplete attribute, so browsers could offer to autofill or save typed values as saved credentials. This sets `autoComplete="off"` on those inputs in both the create and edit forms, matching the `autoComplete="off"` convention already used on the other credential-name inputs. `autoComplete="off"` is a best-effort hint. Browsers may still ignore it for password-typed fields, so this is defense-in-depth hardening, not a hard guarantee that a password manager cannot store the value. --- .server-changes/env-var-inputs-autocomplete-off.md | 6 ++++++ .../route.tsx | 2 ++ .../route.tsx | 1 + 3 files changed, 9 insertions(+) create mode 100644 .server-changes/env-var-inputs-autocomplete-off.md diff --git a/.server-changes/env-var-inputs-autocomplete-off.md b/.server-changes/env-var-inputs-autocomplete-off.md new file mode 100644 index 00000000000..a17457cbd42 --- /dev/null +++ b/.server-changes/env-var-inputs-autocomplete-off.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Stop the browser offering to autofill or save environment variable values as saved credentials. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index 6a87b4b14e7..0204820ff57 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -677,6 +677,7 @@ function VariableField({ onChange={(e) => onChange({ ...value, key: e.currentTarget.value })} autoFocus={index === 0} onPaste={onPaste} + autoComplete="off" /> {fields.key.errors}
@@ -689,6 +690,7 @@ function VariableField({ placeholder="Not set" value={value.value} onChange={(e) => onChange({ ...value, value: e.currentTarget.value })} + autoComplete="off" /> {fields.value.errors}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index 617dc577b80..1974e2051fb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -865,6 +865,7 @@ function EditEnvironmentVariablePanel({ placeholder={variable.isSecret ? "Set new secret value" : "Not set"} defaultValue={variable.value} type={"text"} + autoComplete="off" /> {value.errors} From 00e3c151d4901b4cd5b306729b8531cb2b626b99 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:19:12 +0100 Subject: [PATCH 18/28] feat(webapp): RUN_OPS_SHARDS config, topology and N-way store wiring (#4764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of the RunOps N-way sharding work. This lets the webapp hold N run-ops stores, configured by a single `RUN_OPS_SHARDS` JSON descriptor, and routes to them through the existing keyed router. **Inert with `RUN_OPS_SHARDS` unset** — the topology, the wiring and `ROUTING_ENABLED` are byte-identical to today. ## What's here - **`RUN_OPS_SHARDS`** — a zod-validated JSON array of shard descriptors (`key`, `region`, `url`, `replicaUrl`, `directUrl`, `replication`, `knobs`, `aliasOf`), validated at boot in the `parseMachinePresetCsv` style. Unset or `[]` → no shards. - **One run-ops client factory** — `buildRunOpsWriterClient`/`buildRunOpsReplicaClient` collapse into one `buildRunOpsClient` parameterized by role and resolved pool knobs. The control-plane builders (`buildWriterClient`/`buildReplicaClient`) are a separate path and stay untouched; every resolved value matches the former builders. - **Shard loop in `selectRunOpsTopology`** — one client pair per descriptor; an `aliasOf: "new"` descriptor reuses the new store's clients by reference and opens no pool. - **N-way `buildRunStore`** — builds N dedicated stores + the keyed router via a new `RoutingRunStore.fromShards`, keeping the two-store compat router when no shards are configured. - **`UnknownShardKey`** — raised when an id resolves to an unconfigured key; never falls back to another store. `fromShards` injects `resolveShard` so a gen-2 id routes to its own shard. - **Per-shard transaction resilience** — each shard gets its own retry budget. - **Mint bound** — `computeMintShard` intersects the active mint list with the configured descriptor keys, so a key with no descriptor is never minted into. - **Boot table** — logs `key`, address fingerprint (host:port/db, no credentials), and role, only when shards are configured. ## Ordering constraint Do **not** configure a `RUN_OPS_SHARDS` descriptor in any environment until the routing-semantics change (TRI-13427) lands — three fan-out sites still truncate at N>2. Merging this PR alone is safe (inert with the var unset); configuring a descriptor is what must wait. ## Testing - Run-store corpus: green with zero test-file diffs (the bit-identical proof for the compat router). - `runOpsDbTopology.test.ts` 17/17, `runStore.server.test.ts` 4/4, `runOpsMigration` family 149/149. - New unit suites: descriptor validation, pool-knob value tables, `fromShards` routing + `UnknownShardKey`, boot-table formatter, mint bound. - typecheck (webapp + run-store), knip, lint, format: pass. ## Changelog Internal run-ops sharding infrastructure. No changeset or `.server-changes`: the change is inert with `RUN_OPS_SHARDS` unset and has no user-visible behaviour. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 338 ++++++++++-------- apps/webapp/app/env.server.ts | 11 + .../mintShardAssignment.test.ts | 37 ++ .../v3/runOpsMigration/mintShardAssignment.ts | 18 +- .../runOpsMigration/runOpsMintShard.server.ts | 2 + apps/webapp/app/v3/runOpsPoolKnobs.server.ts | 80 +++++ apps/webapp/app/v3/runOpsShardTable.ts | 28 ++ apps/webapp/app/v3/runOpsShards.server.ts | 124 +++++++ apps/webapp/app/v3/runStore.server.ts | 39 +- .../app/v3/transactionResilience.server.ts | 45 ++- apps/webapp/test/runOpsDbTopology.test.ts | 69 ++++ apps/webapp/test/runOpsPoolKnobs.test.ts | 42 +++ apps/webapp/test/runOpsShardBootTable.test.ts | 34 ++ apps/webapp/test/runOpsShards.test.ts | 84 +++++ apps/webapp/test/runStoreShardWiring.test.ts | 44 +++ .../webapp/test/transactionResilience.test.ts | 17 + .../core/src/v3/isomorphic/friendlyId.test.ts | 16 + packages/core/src/v3/isomorphic/friendlyId.ts | 5 + 18 files changed, 874 insertions(+), 159 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsPoolKnobs.server.ts create mode 100644 apps/webapp/app/v3/runOpsShardTable.ts create mode 100644 apps/webapp/app/v3/runOpsShards.server.ts create mode 100644 apps/webapp/test/runOpsPoolKnobs.test.ts create mode 100644 apps/webapp/test/runOpsShardBootTable.test.ts create mode 100644 apps/webapp/test/runOpsShards.test.ts create mode 100644 apps/webapp/test/runStoreShardWiring.test.ts create mode 100644 apps/webapp/test/transactionResilience.test.ts diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index a69c83cd375..21a9cb9fe22 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -31,15 +31,18 @@ import { assertSplitRealtimeInterlock, } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; -import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; -import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; +import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; +import { buildRunOpsShardTable } from "./v3/runOpsShardTable"; import { + resolveShardResilience, controlPlaneTransactionResilience, registerTransactionResilience, resilienceForClient, runOpsLegacyTransactionResilience, runOpsTransactionResilience, } from "./v3/transactionResilience.server"; +import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; +import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import type { Span } from "@opentelemetry/api"; import { context, trace } from "@opentelemetry/api"; import { queryPerformanceMonitor } from "./utils/queryPerformanceMonitor.server"; @@ -275,10 +278,19 @@ export const webhookReplica: WebhookReplicaDatabase = singleton("webhookReplica" type RunOpsClients = { writer: PrismaClient; replica: PrismaReplicaClient }; type NewRunOpsClients = { writer: RunOpsPrismaClient; replica: RunOpsPrismaClient }; +type ShardTopologyDescriptor = { + key: string; + url?: string; + replicaUrl?: string; + aliasOf?: "new"; +}; export type RunOpsTopology = { newRunOps: NewRunOpsClients; legacyRunOps: RunOpsClients; controlPlane: RunOpsClients; + // One client pair per gen-2 shard descriptor. Empty unless RUN_OPS_SHARDS is configured. An + // aliasOf:"new" descriptor maps to the newRunOps pair BY REFERENCE (no new pool). + shards: Map; }; export type SelectRunOpsTopologyConfig = { splitEnabled: boolean; @@ -288,6 +300,7 @@ export type SelectRunOpsTopologyConfig = { newReplicaUrl?: string; // When true, legacy reuses the control-plane client instead of opening its own pool. Defaults to false. legacySharesControlPlane?: boolean; + shards?: ShardTopologyDescriptor[]; }; export type RunOpsClientBuilders = { controlPlane: RunOpsClients; @@ -297,6 +310,10 @@ export type RunOpsClientBuilders = { // RunOpsPrismaClient double-cast needed): the legacy DB carries the full control-plane schema. buildLegacyWriter: (url: string, clientType: string) => PrismaClient; buildLegacyReplica: (url: string, clientType: string) => PrismaReplicaClient; + // Receive the whole descriptor so the singleton can resolve per-shard knobs and resilience by key. + // Optional so the existing test literals (which build no shards) need no change. + buildShardWriter?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; + buildShardReplica?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; }; // Pure run-ops client selector. No env, no isSplitEnabled() — those @@ -315,11 +332,11 @@ export function selectRunOpsTopology( }; if (!config.splitEnabled) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } if (!config.legacyUrl || !config.newUrl) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } // Same-DB legacy reuses the control-plane pool; only build a separate pool once the DSNs diverge. @@ -338,12 +355,28 @@ export function selectRunOpsTopology( const newReplica: RunOpsPrismaClient = config.newReplicaUrl ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica") : newWriter; + const newRunOps: NewRunOpsClients = { writer: newWriter, replica: newReplica }; + + const shards = new Map(); + for (const shard of config.shards ?? []) { + if (shard.aliasOf === "new") { + // Aliased: share the new store's clients by reference. No builder, no new pool — the soak path. + shards.set(shard.key, newRunOps); + continue; + } + if (!shard.url || !builders.buildShardWriter || !builders.buildShardReplica) { + throw new Error( + `selectRunOpsTopology: shard "${shard.key}" needs a url and shard builders when not aliased` + ); + } + const shardWriter = builders.buildShardWriter(shard); + const shardReplica: RunOpsPrismaClient = shard.replicaUrl + ? builders.buildShardReplica(shard) + : shardWriter; + shards.set(shard.key, { writer: shardWriter, replica: shardReplica }); + } - return { - newRunOps: { writer: newWriter, replica: newReplica }, - legacyRunOps, - controlPlane, - }; + return { newRunOps, legacyRunOps, controlPlane, shards }; } // The env-bound run-ops topology singleton. The split decision uses @@ -376,6 +409,17 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ); } + const newPoolKnobs = resolveRunOpsPoolKnobs("new"); + const shardDescriptorsByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d])); + + // Boot table: emit ONLY when shards are configured, so the inert (RUN_OPS_SHARDS unset) merge adds + // no new log output. The fingerprint is an address, not an identity claim (see runOpsAddressFingerprint). + if (env.RUN_OPS_SHARDS.length > 0) { + logger.info("run-ops shard topology (fingerprint is an address, NOT an identity claim)", { + shards: buildRunOpsShardTable(env.RUN_OPS_SHARDS), + }); + } + return selectRunOpsTopology( { splitEnabled, @@ -384,6 +428,12 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { newUrl, newReplicaUrl: env.RUN_OPS_DATABASE_READ_REPLICA_URL, legacySharesControlPlane, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + url: d.url, + replicaUrl: d.replicaUrl, + aliasOf: d.aliasOf, + })), }, { controlPlane: { writer: prisma, replica: $replica }, @@ -392,10 +442,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-writer", - buildRunOpsWriterClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + role: "writer", + connectionLimit: newPoolKnobs.connectionLimit, + poolTimeout: newPoolKnobs.writerPoolTimeout, + connectTimeout: newPoolKnobs.writerConnectionTimeout, + useDriverAdapter: newPoolKnobs.writerDriverAdapter, }) ) ), @@ -409,10 +463,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-replica", - buildRunOpsReplicaClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + role: "replica", + connectionLimit: newPoolKnobs.replicaConnectionLimit, + poolTimeout: newPoolKnobs.replicaPoolTimeout, + connectTimeout: newPoolKnobs.replicaConnectionTimeout, + useDriverAdapter: newPoolKnobs.replicaDriverAdapter, }) ) ) @@ -450,6 +508,50 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ) ) ), + // A gen-2 shard is a dedicated run-ops DB, so it mirrors buildNewWriter/buildNewReplica: same + // client class, same wrapper stack, its OWN resilience budget, and the "new"-role pool knobs + // merged with the descriptor's per-shard overrides. Shards share the run-ops datasource tag. + buildShardWriter: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return registerTransactionResilience( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-writer", + buildRunOpsClient({ + url: shard.url!, + clientType: `run-ops-shard-${shard.key}-writer`, + role: "writer", + connectionLimit: knobs.connectionLimit, + poolTimeout: knobs.writerPoolTimeout, + connectTimeout: knobs.writerConnectionTimeout, + useDriverAdapter: knobs.writerDriverAdapter, + }) + ) + ), + resolveShardResilience(shard.key, descriptor?.knobs) + ); + }, + buildShardReplica: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return markReadReplicaClient( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-replica", + buildRunOpsClient({ + url: shard.replicaUrl!, + clientType: `run-ops-shard-${shard.key}-replica`, + role: "replica", + connectionLimit: knobs.replicaConnectionLimit, + poolTimeout: knobs.replicaPoolTimeout, + connectTimeout: knobs.replicaConnectionTimeout, + useDriverAdapter: knobs.replicaDriverAdapter, + }) + ) + ) + ); + }, } ); }); @@ -475,6 +577,22 @@ export const runOpsLegacyPrismaClient: RunOpsPrismaClient = runOpsTopology.legac export const runOpsLegacyReplicaClient: RunOpsPrismaClient = runOpsTopology.legacyRunOps .replica as unknown as RunOpsPrismaClient; +// Gen-2 shard handles for the run-store boundary. Empty unless RUN_OPS_SHARDS is configured. +// `aliasOf` carries the descriptor's declared alias so the router can dedup an aliased shard (which +// shares its target's database) out of every fan-out sum — the DECLARATION, not client identity. +const runOpsShardAliasByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d.aliasOf])); +export const runOpsShardHandles: Array<{ + key: string; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + aliasOf?: string; +}> = [...runOpsTopology.shards.entries()].map(([key, clients]) => ({ + key, + writer: clients.writer, + replica: clients.replica, + aliasOf: runOpsShardAliasByKey.get(key), +})); + export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ newReplica: runOpsNewReplicaClient, controlPlaneWriter: prisma, @@ -924,64 +1042,64 @@ export function buildReplicaClient({ return replicaClient; } -function buildRunOpsWriterClient({ +// One factory for the run-ops writer and replica clients, backed by the dedicated RunOpsPrismaClient +// (a separately generated Prisma package). Parameterized by role and the resolved pool knobs, so a +// gen-1 new store and every gen-2 shard share this single builder. The control-plane builders +// (buildWriterClient/buildReplicaClient) are a DIFFERENT path and are untouched — this reuses only +// the shared low-level helpers (buildPrismaConnectionUrl, buildDriverAdapterPool). +function buildRunOpsClient({ url, clientType, + role, + connectionLimit, + poolTimeout, + connectTimeout, useDriverAdapter = false, }: { url: string; clientType: string; + role: "writer" | "replica"; + connectionLimit: number; + poolTimeout: number; + connectTimeout: number; useDriverAdapter?: boolean; }): RunOpsPrismaClient { - const databaseUrl = buildPrismaConnectionUrl(url, { - connectionLimit: env.DATABASE_CONNECTION_LIMIT.toString(), - poolTimeout: (env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), + const isWriter = role === "writer"; + const setupLabel = isWriter ? "run-ops prisma client" : "run-ops read replica connection"; + const connectedLabel = isWriter + ? "run-ops prisma client connected" + : "run-ops read replica connected"; + + const connectionUrl = buildPrismaConnectionUrl(url, { + connectionLimit: connectionLimit.toString(), + poolTimeout: poolTimeout.toString(), + connectTimeout: connectTimeout.toString(), applicationName: env.SERVICE_NAME, }); console.log( - `🔌 setting up run-ops prisma client to ${redactUrlSecrets(databaseUrl)}${ + `🔌 setting up ${setupLabel} to ${redactUrlSecrets(connectionUrl)}${ useDriverAdapter ? " (pg driver adapter)" : "" }` ); + const log = [ + { emit: "event", level: "error" }, + { emit: "event", level: "info" }, + { emit: "event", level: "warn" }, + ...((process.env.VERBOSE_PRISMA_LOGS === "1" || + process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined + ? [{ emit: "event", level: "query" }] + : []) as { emit: "event"; level: "query" }[]), + ] as const; + const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ) + ? buildDriverAdapterPool(url, clientType, poolTimeout, connectionLimit) : undefined; const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }) - : new RunOpsPrismaClient({ - datasources: { db: { url: databaseUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); + ? new RunOpsPrismaClient({ adapter: driverPool.adapter, log: [...log] }) + : new RunOpsPrismaClient({ datasources: { db: { url: connectionUrl.href } }, log: [...log] }); registerDatabaseMetricsSource( driverPool @@ -999,117 +1117,27 @@ function buildRunOpsWriterClient({ client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log, ignoreError: true }) - ); - } - - client.$on("query", (log) => queryPerformanceMonitor.onQuery("writer", log)); - - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (writer)", { error }); - }); - } - - console.log(`🔌 run-ops prisma client connected`); - - return client; -} - -function buildRunOpsReplicaClient({ - url, - clientType, - useDriverAdapter = false, -}: { - url: string; - clientType: string; - useDriverAdapter?: boolean; -}): RunOpsPrismaClient { - const replicaUrl = buildPrismaConnectionUrl(url, { - connectionLimit: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ).toString(), - poolTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), - applicationName: env.SERVICE_NAME, - }); - - console.log( - `🔌 setting up run-ops read replica connection to ${redactUrlSecrets(replicaUrl)}${ - useDriverAdapter ? " (pg driver adapter)" : "" - }` - ); - - const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, + // The writer bridges P2002 -> 422 at the store boundary, so its infra errors are logged once + // there (ignoreError). Replica errors are not on that write path, so they log normally. + logger.error("RunOpsPrismaClient error", { clientType, - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ) - : undefined; - - const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], + event: log, + ...(isWriter ? { ignoreError: true } : {}), }) - : new RunOpsPrismaClient({ - datasources: { db: { url: replicaUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); - - registerDatabaseMetricsSource( - driverPool - ? { - clientType, - usesDriverAdapter: true, - client, - pool: driverPool.pool, - poolCounters: driverPool.poolCounters, - } - : { clientType, usesDriverAdapter: false, client } - ); - - if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { - client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); - client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); - client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log }) ); } - client.$on("query", (log) => queryPerformanceMonitor.onQuery("replica", log)); + client.$on("query", (log) => queryPerformanceMonitor.onQuery(role, log)); - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (replica)", { error }); - }); - } + // Eager connect is a warm-up only — Prisma reconnects lazily on first query. ALWAYS catch the + // rejection (not just under NODE_ENV=test), so a shard/run-ops DB that is unreachable at boot + // logs a warning instead of surfacing as an unhandled promise rejection. One unreachable shard + // must not take down webapp startup. + client.$connect().catch((error) => { + logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); + }); - console.log(`🔌 run-ops read replica connected`); + console.log(`🔌 ${connectedLabel}`); return client; } diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index aec9cd82927..c1155d377d3 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { MachinePresetName } from "@trigger.dev/core/v3"; import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; import { isValidDuration } from "./services/realtime/duration.server"; @@ -310,6 +311,8 @@ const EnvironmentSchema = z RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), + // Gen-2 shard descriptors as a JSON array. Unset/"" -> [] (today). See runOpsShards.server.ts. + RUN_OPS_SHARDS: z.string().optional().transform(parseRunOpsShards), // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), @@ -2491,6 +2494,14 @@ const EnvironmentSchema = z }); } } + if (!validateShardListAgainstNewUrl(env.RUN_OPS_SHARDS, env.RUN_OPS_DATABASE_URL)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["RUN_OPS_SHARDS"], + message: + "RUN_OPS_SHARDS is non-empty but RUN_OPS_DATABASE_URL is unset; a shard requires the gen-1 new store", + }); + } }); export type Environment = z.infer; diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts index d88e64e1d75..a4e4a64d36d 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -473,3 +473,40 @@ describe("computeMintShard — the global override wins the complete cutover", ( ); }); }); + +describe("routableKeys bound (the shard descriptor keys this deployment can route)", () => { + it("drops an active key that is not routable, so the hash never returns it", () => { + // "z" is in the active list but not configured as a descriptor -> only "a" is selectable. + const ids = envIds(200); + for (const id of ids) { + const shard = computeMintShard({ id }, deps({ set: ["a", "z"] }, { routableKeys: ["a"] })); + expect(shard).toBe("a"); + } + }); + + it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] }))).toBe( + "new" + ); + }); + + it("rejects a per-org pin to a non-routable key and falls through to the hash", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps({ set: ["a", "z"] }, { ...orgFlags({ runOpsMintShard: "z" }), routableKeys: ["a"] }) + ); + expect(shard).toBe("a"); + }); + + it("with no routableKeys given, behaviour is unchanged", () => { + const ids = envIds(200); + for (const id of ids) { + const withBound = computeMintShard( + { id }, + deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] }) + ); + const without = computeMintShard({ id }, deps({ set: ["a", "b"] })); + expect(withBound).toBe(without); + } + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts index a49a1a6a60d..2855f936249 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts @@ -19,6 +19,10 @@ export type MintShardDeps = { nowMs: number; graceMs: number; orgFeatureFlags: unknown; + // The shard keys this deployment can actually route (the RUN_OPS_SHARDS descriptor keys). The + // active set is bounded to these, so a stored key with no descriptor is never minted into. + // Undefined means "no bound" (today's behaviour). + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; }; @@ -94,7 +98,17 @@ function hrwSelect(environmentId: string, activeSet: string[]): string { // would leak the drain the active list performs, and throwing would fail customer triggers // whenever a pinned shard drains. export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey { - const activeSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + const rawActiveSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + // Empty check BEFORE the bound, so an unconfigured deployment returns "new" exactly as today. + if (rawActiveSet.length === 0) { + return "new"; + } + + // Bound the active set to the keys this deployment can route. A stored key with no descriptor is + // dropped, never minted into. If nothing survives, fall back to gen-1 (fail-safe, never a throw). + const activeSet = deps.routableKeys + ? rawActiveSet.filter((key) => deps.routableKeys!.includes(key)) + : rawActiveSet; if (activeSet.length === 0) { return "new"; } @@ -148,6 +162,7 @@ export type ResolveMintShardDeps = { ttlMs: number; graceMs: number; orgFeatureFlags: unknown; + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; onReadFailed?: (error: unknown) => void; @@ -200,6 +215,7 @@ export async function resolveMintShardWith( nowMs: deps.nowMs, graceMs: deps.graceMs, orgFeatureFlags: deps.orgFeatureFlags, + routableKeys: deps.routableKeys, onPinRejected: deps.onPinRejected, onOverrideRejected: deps.onOverrideRejected, }); diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts index 542384e16f8..c1c2b9ddd48 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -84,6 +84,8 @@ export async function resolveMintShard(environment: { ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS, graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, orgFeatureFlags: environment.orgFeatureFlags, + // Bound the active list to the shards this deployment can actually route. + routableKeys: env.RUN_OPS_SHARDS.map((shard) => shard.key), onPinRejected: reportPinRejected, onOverrideRejected: reportOverrideRejected, onReadFailed: (error) => diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts new file mode 100644 index 00000000000..0396af0f6d2 --- /dev/null +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -0,0 +1,80 @@ +import { env } from "~/env.server"; +import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; + +// Pool configuration for one run-ops store (writer + replica). Kept separate from db.server (which +// ~156 tests mock wholesale) so a new export breaks no mock. +export type ResolvedPoolKnobs = { + writerPoolTimeout: number; + writerConnectionTimeout: number; + writerDriverAdapter: boolean; + connectionLimit: number; + replicaConnectionLimit: number; + replicaPoolTimeout: number; + replicaConnectionTimeout: number; + replicaDriverAdapter: boolean; +}; + +type Role = "new" | "legacy"; + +// PURE: overlay a gen-2 shard's descriptor knobs on a role's resolved defaults. This holds the only +// logic (per-field override), so a test drives it with literal defaults and literal overrides — +// no env import, no circular assertion against the same env expression the impl reads. +export function applyPoolKnobOverrides( + defaults: ResolvedPoolKnobs, + k?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return { + writerPoolTimeout: k?.writerPoolTimeout ?? defaults.writerPoolTimeout, + writerConnectionTimeout: k?.writerConnectionTimeout ?? defaults.writerConnectionTimeout, + writerDriverAdapter: k?.writerDriverAdapter ?? defaults.writerDriverAdapter, + connectionLimit: k?.connectionLimit ?? defaults.connectionLimit, + replicaConnectionLimit: k?.replicaConnectionLimit ?? defaults.replicaConnectionLimit, + replicaPoolTimeout: k?.replicaPoolTimeout ?? defaults.replicaPoolTimeout, + replicaConnectionTimeout: k?.replicaConnectionTimeout ?? defaults.replicaConnectionTimeout, + replicaDriverAdapter: k?.replicaDriverAdapter ?? defaults.replicaDriverAdapter, + }; +} + +// The env-derived defaults for a role, reproducing today's run-ops builder expressions exactly. A +// flat mapping (no logic), verified by inspection against the former builders. Transaction +// resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. +function poolKnobDefaults(role: Role): ResolvedPoolKnobs { + if (role === "legacy") { + return { + writerPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; + } + + return { + writerPoolTimeout: env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; +} + +export function resolveRunOpsPoolKnobs( + role: Role, + descriptorKnobs?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return applyPoolKnobOverrides(poolKnobDefaults(role), descriptorKnobs); +} diff --git a/apps/webapp/app/v3/runOpsShardTable.ts b/apps/webapp/app/v3/runOpsShardTable.ts new file mode 100644 index 00000000000..6741b3cb5b0 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShardTable.ts @@ -0,0 +1,28 @@ +// Pure boot-table helpers. Dependency-free (no db.server, no env) so a test of these two string +// functions never constructs a Prisma client. db.server imports them for the boot log. + +// A host:port/db address, with NO username and NO query params — never a secret, and deliberately +// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the +// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. +export function runOpsAddressFingerprint(url: string): string { + try { + const u = new URL(url); + return `${u.hostname}:${u.port || "5432"}${u.pathname}`; + } catch { + return "unparseable"; + } +} + +export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; + +// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and +// carries no address (it shares the new store's pool). +export function buildRunOpsShardTable( + descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> +): RunOpsShardTableRow[] { + return descriptors.map((d) => + d.aliasOf + ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } + : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } + ); +} diff --git a/apps/webapp/app/v3/runOpsShards.server.ts b/apps/webapp/app/v3/runOpsShards.server.ts new file mode 100644 index 00000000000..23c45efa404 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShards.server.ts @@ -0,0 +1,124 @@ +import { z } from "zod"; +import { isValidShardChar } from "@trigger.dev/core/v3/isomorphic"; +import { isValidDatabaseUrl } from "~/utils/db"; + +const KnobsSchema = z + .object({ + writerPoolTimeout: z.number().int().optional(), + writerConnectionTimeout: z.number().int().optional(), + writerDriverAdapter: z.boolean().optional(), + connectionLimit: z.number().int().optional(), + replicaConnectionLimit: z.number().int().optional(), + replicaPoolTimeout: z.number().int().optional(), + replicaConnectionTimeout: z.number().int().optional(), + replicaDriverAdapter: z.boolean().optional(), + transactionMaxWaitMs: z.number().int().optional(), + transactionStartRetryEnabled: z.boolean().optional(), + transactionStartRetryMaxAttempts: z.number().int().optional(), + transactionStartRetryBackoffMinMs: z.number().int().optional(), + transactionStartRetryBackoffMaxMs: z.number().int().optional(), + transactionStartRetryBudgetPerSec: z.number().int().optional(), + transactionStartRetryBudgetBurst: z.number().int().optional(), + }) + .strict(); +export type RunOpsShardKnobs = z.infer; + +const ReplicationSchema = z.object({ + slotName: z.string().min(1), + publicationName: z.string().min(1), + originGeneration: z.number().int().min(2).max(255), +}); + +const DescriptorSchema = z + .object({ + key: z.string().refine(isValidShardChar, "shard key must be a single [a-z0-9] char"), + region: z.string().min(1), + url: z.string().refine(isValidDatabaseUrl, "url is invalid").optional(), + replicaUrl: z.string().refine(isValidDatabaseUrl, "replicaUrl is invalid").optional(), + directUrl: z.string().refine(isValidDatabaseUrl, "directUrl is invalid").optional(), + replication: ReplicationSchema.optional(), + knobs: KnobsSchema.optional(), + aliasOf: z.literal("new").optional(), + }) + .strict() + .superRefine((d, ctx) => { + const hasUrl = d.url !== undefined; + const hasAlias = d.aliasOf !== undefined; + if (hasUrl === hasAlias) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "exactly one of url or aliasOf is required", + }); + } + if (!hasAlias && d.replication === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "replication is required unless aliasOf is set", + }); + } + }); + +export type RunOpsShardDescriptor = z.infer; + +// Boot-validated transform, in the style of parseMachinePresetCsv. Undefined and "" both mean the +// off state and resolve to []. The undefined guard is load-bearing: an unguarded JSON.parse would +// kill every single-DB boot, which never sets this variable. +export function parseRunOpsShards( + raw: string | undefined, + ctx: z.RefinementCtx +): RunOpsShardDescriptor[] { + if (raw === undefined || raw.trim() === "") return []; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "RUN_OPS_SHARDS is not valid JSON" }); + return z.NEVER; + } + + const result = z.array(DescriptorSchema).safeParse(parsed); + if (!result.success) { + for (const issue of result.error.issues) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS[${issue.path.join(".")}]: ${issue.message}`, + }); + } + return z.NEVER; + } + + const keys = new Set(); + const gens = new Set(); + for (const d of result.data) { + if (keys.has(d.key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate key ${d.key}`, + }); + return z.NEVER; + } + keys.add(d.key); + if (d.replication) { + if (gens.has(d.replication.originGeneration)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate originGeneration ${d.replication.originGeneration}`, + }); + return z.NEVER; + } + gens.add(d.replication.originGeneration); + } + } + + return result.data; +} + +// A non-empty shard list requires the gen-1 new store, because gen-1 v1 ids resolve to "new" +// forever (append-only). Pure so the boot refinement and its test share one rule. +export function validateShardListAgainstNewUrl( + shards: RunOpsShardDescriptor[], + newUrl: string | undefined +): boolean { + return shards.length === 0 || !!newUrl; +} diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 15e860e65c3..7f51a940cec 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -16,6 +16,7 @@ import { runOpsLegacyReplica, runOpsNewPrismaClient, runOpsNewReplicaClient, + runOpsShardHandles, } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; @@ -38,6 +39,16 @@ type BuildRunStoreDeps = { singleReplica: PrismaReplicaClient; /** Id-to-shard-key resolver; defaults to the core resolveShard inside RoutingRunStore. */ resolveShard?: (id: string) => ShardKey; + /** Gen-2 shard handles. When non-empty, buildRunStore builds one dedicated store per descriptor and + * hands them to the N-way router; an aliased shard shares its target's database. Empty/absent keeps + * the two-store compat router. */ + shards?: Array<{ + key: ShardKey; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + aliasOf?: ShardKey; + resilience?: TransactionResilienceConfig; + }>; /** Per-pool transaction-resilience configs threaded into the store(s) this builds (IoC). */ singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; @@ -86,9 +97,25 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { transactionStartRetry: deps.legacyResilience?.startRetry, }); + // Gen-2 shards: one dedicated store per descriptor, handed to the N-way router. An aliased shard + // builds a store over its target's (shared) client and carries aliasOf, so the router dedups it out + // of fan-out sums by declaration. No shards -> the two-store compat router (byte-identical). + const shardStores = (deps.shards ?? []).map((shard) => ({ + key: shard.key, + store: new PostgresRunStore({ + prisma: shard.writer, + readOnlyPrisma: shard.replica, + schemaVariant: "dedicated" as const, + maxWait: shard.resilience?.maxWait, + transactionStartRetry: shard.resilience?.startRetry, + }), + aliasOf: shard.aliasOf, + })); + return new RoutingRunStore({ new: newStore, legacy: legacyStore, + shards: shardStores.length > 0 ? shardStores : undefined, resolveShard: deps.resolveShard ?? resolveShard, metrics: routingStoreMetrics, }); @@ -138,6 +165,8 @@ function tryResolveRunOpsHandles() { newReplica: runOpsNewReplicaClient, legacyWriter: runOpsLegacyPrisma, legacyReplica: runOpsLegacyReplica, + // Absent under a minimal db.server mock; default to no shards so the compat router is built. + shardHandles: runOpsShardHandles ?? [], }; } catch { return null; @@ -155,9 +184,17 @@ export const runStore: RunStore = singleton("RunStore", () => { singleResilience: resilienceForClient(prisma), }); } + const { shardHandles, ...storeHandles } = handles; return buildRunStore({ splitEnabled: true, - ...handles, + ...storeHandles, + shards: shardHandles.map((shard) => ({ + key: shard.key, + writer: shard.writer, + replica: shard.replica, + aliasOf: shard.aliasOf, + resilience: resilienceForClient(shard.writer), + })), singleWriter: prisma, singleReplica: $replica, singleResilience: resilienceForClient(prisma), diff --git a/apps/webapp/app/v3/transactionResilience.server.ts b/apps/webapp/app/v3/transactionResilience.server.ts index ae5678c987c..eabde6691d8 100644 --- a/apps/webapp/app/v3/transactionResilience.server.ts +++ b/apps/webapp/app/v3/transactionResilience.server.ts @@ -17,8 +17,11 @@ export type TransactionResilienceConfig = { startRetry: TransactionStartRetryConfig; }; -function resolveTransactionResilience( - pool: "control-plane" | "run-ops" | "run-ops-legacy", +// Exported so the topology singleton can build a per-shard config (each call creates its OWN +// TokenBucketRetryBudget, so one shard's retry storm cannot drain another's). `pool` is a free +// string — it only labels a log line, never keys any behaviour. +export function resolveTransactionResilience( + pool: string, overrides: { maxWaitMs?: number; enabled?: boolean; @@ -64,6 +67,44 @@ export const runOpsTransactionResilience = resolveTransactionResilience("run-ops budgetBurst: env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, }); +// A gen-2 shard's resilience. Defaults to the RUN_OPS_DATABASE_TRANSACTION_* values (so a shard with +// no overrides matches the gen-1 new store), then applies the descriptor's per-shard overrides. Each +// call builds its OWN budget, so a storm on one shard cannot drain another's. +export function resolveShardResilience( + key: string, + overrides?: { + transactionMaxWaitMs?: number; + transactionStartRetryEnabled?: boolean; + transactionStartRetryMaxAttempts?: number; + transactionStartRetryBackoffMinMs?: number; + transactionStartRetryBackoffMaxMs?: number; + transactionStartRetryBudgetPerSec?: number; + transactionStartRetryBudgetBurst?: number; + } +): TransactionResilienceConfig { + return resolveTransactionResilience(`run-ops-shard-${key}`, { + maxWaitMs: overrides?.transactionMaxWaitMs ?? env.RUN_OPS_DATABASE_TRANSACTION_MAX_WAIT_MS, + enabled: + overrides?.transactionStartRetryEnabled ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_ENABLED, + maxAttempts: + overrides?.transactionStartRetryMaxAttempts ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_MAX_ATTEMPTS, + backoffMinMs: + overrides?.transactionStartRetryBackoffMinMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MIN_MS, + backoffMaxMs: + overrides?.transactionStartRetryBackoffMaxMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MAX_MS, + budgetPerSec: + overrides?.transactionStartRetryBudgetPerSec ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_PER_SEC, + budgetBurst: + overrides?.transactionStartRetryBudgetBurst ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, + }); +} + export const runOpsLegacyTransactionResilience = resolveTransactionResilience("run-ops-legacy", { maxWaitMs: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_MAX_WAIT_MS, enabled: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_START_RETRY_ENABLED, diff --git a/apps/webapp/test/runOpsDbTopology.test.ts b/apps/webapp/test/runOpsDbTopology.test.ts index 8890fdbb662..f2bcc0bf5ea 100644 --- a/apps/webapp/test/runOpsDbTopology.test.ts +++ b/apps/webapp/test/runOpsDbTopology.test.ts @@ -142,6 +142,75 @@ describe("selectRunOpsTopology (pure)", () => { expect(topo.legacyRunOps.replica).toBe(legacyWriter); expect(buildLegacyReplica).not.toHaveBeenCalled(); }); + + const baseSplit = { + splitEnabled: true, + legacyUrl: "postgres://legacy", + newUrl: "postgres://new", + }; + const baseBuilders = () => ({ + controlPlane: cp, + buildNewWriter: vi.fn().mockReturnValue({ tag: "nw" } as any), + buildNewReplica: vi.fn().mockReturnValue({ tag: "nr" } as any), + buildLegacyWriter: vi.fn().mockReturnValue({ tag: "lw" } as any), + buildLegacyReplica: vi.fn().mockReturnValue({ tag: "lr" } as any), + }); + + it("no descriptors: the shards map is empty", () => { + const topo = selectRunOpsTopology(baseSplit, baseBuilders()); + expect(topo.shards.size).toBe(0); + }); + + it("two descriptors: two shard client pairs, each built once", () => { + const buildShardWriter = vi.fn((s: any) => ({ tag: `w:${s.key}` }) as any); + const buildShardReplica = vi.fn((s: any) => ({ tag: `r:${s.key}` }) as any); + const topo = selectRunOpsTopology( + { + ...baseSplit, + shards: [ + { key: "a", url: "postgres://a", replicaUrl: "postgres://a-r" }, + { key: "b", url: "postgres://b" }, + ], + }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.size).toBe(2); + expect(topo.shards.get("a")!.writer).toEqual({ tag: "w:a" }); + // b has no replicaUrl, so its replica falls back to its writer (buildShardReplica not called for b). + expect(topo.shards.get("b")!.replica).toEqual({ tag: "w:b" }); + expect(buildShardWriter).toHaveBeenCalledTimes(2); + expect(buildShardReplica).toHaveBeenCalledTimes(1); + }); + + it("an alias descriptor reuses newRunOps by reference and calls no shard builder", () => { + const buildShardWriter = vi.fn(); + const buildShardReplica = vi.fn(); + const topo = selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", aliasOf: "new" }] }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.get("a")).toBe(topo.newRunOps); + expect(buildShardWriter).not.toHaveBeenCalled(); + expect(buildShardReplica).not.toHaveBeenCalled(); + }); + + it("throws when a non-aliased shard has no url (guards the shard.url non-null assertion)", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a" }] }, + { ...baseBuilders(), buildShardWriter: vi.fn(), buildShardReplica: vi.fn() } + ) + ).toThrow(/shard "a" needs a url/); + }); + + it("throws when a non-aliased shard is configured but the shard builders are absent", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", url: "postgres://a" }] }, + baseBuilders() + ) + ).toThrow(/shard "a" needs a url and shard builders/); + }); }); describe("sameDatabaseTarget", () => { diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts new file mode 100644 index 00000000000..38353682cb2 --- /dev/null +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { applyPoolKnobOverrides, type ResolvedPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; + +// Literal defaults, so the assertions lock the override logic against fixed values rather than +// against the same env expression the implementation reads. No env import (webapp test rule). +const DEFAULTS: ResolvedPoolKnobs = { + writerPoolTimeout: 10, + writerConnectionTimeout: 20, + writerDriverAdapter: false, + connectionLimit: 30, + replicaConnectionLimit: 40, + replicaPoolTimeout: 50, + replicaConnectionTimeout: 60, + replicaDriverAdapter: false, +}; + +describe("applyPoolKnobOverrides", () => { + it("returns the defaults verbatim when no descriptor knobs are given", () => { + expect(applyPoolKnobOverrides(DEFAULTS)).toEqual(DEFAULTS); + expect(applyPoolKnobOverrides(DEFAULTS, {})).toEqual(DEFAULTS); + }); + + it("overrides only the fields the descriptor sets", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { + connectionLimit: 999, + writerDriverAdapter: true, + replicaPoolTimeout: 555, + }); + expect(result.connectionLimit).toBe(999); + expect(result.writerDriverAdapter).toBe(true); + expect(result.replicaPoolTimeout).toBe(555); + // Untouched fields keep the defaults. + expect(result.writerPoolTimeout).toBe(10); + expect(result.replicaConnectionLimit).toBe(40); + expect(result.replicaDriverAdapter).toBe(false); + }); + + it("does not read the transaction knobs off the descriptor", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { transactionMaxWaitMs: 1234 }); + expect(result).toEqual(DEFAULTS); + }); +}); diff --git a/apps/webapp/test/runOpsShardBootTable.test.ts b/apps/webapp/test/runOpsShardBootTable.test.ts new file mode 100644 index 00000000000..3031608d701 --- /dev/null +++ b/apps/webapp/test/runOpsShardBootTable.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/v3/runOpsShardTable"; + +describe("runOpsAddressFingerprint", () => { + it("returns host:port/db with no username or query params", () => { + const fp = runOpsAddressFingerprint( + "postgres://user:pw@host.example:5433/mydb?schema=public&pool_timeout=20" + ); + expect(fp).toBe("host.example:5433/mydb"); + expect(fp).not.toContain("user"); + expect(fp).not.toContain("pool_timeout"); + }); + it("defaults the port to 5432", () => { + expect(runOpsAddressFingerprint("postgres://h/db")).toBe("h:5432/db"); + }); + it("returns a marker on unparseable input rather than throwing", () => { + expect(runOpsAddressFingerprint("not a url")).toBe("unparseable"); + }); +}); + +describe("buildRunOpsShardTable", () => { + it("one row per descriptor, with key, fingerprint, and role", () => { + const rows = buildRunOpsShardTable([ + { key: "a", url: "postgres://user:pw@h/adb?schema=public" }, + { key: "b", aliasOf: "new" }, + ]); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual({ key: "a", fingerprint: "h:5432/adb", role: "shard" }); + expect(rows[1]).toEqual({ key: "b", fingerprint: "alias(new)", role: "alias(new)" }); + }); + it("is empty for an empty descriptor list", () => { + expect(buildRunOpsShardTable([])).toEqual([]); + }); +}); diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts new file mode 100644 index 00000000000..fef7e925e75 --- /dev/null +++ b/apps/webapp/test/runOpsShards.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; + +function run(raw: string | undefined) { + const schema = z.string().optional().transform(parseRunOpsShards); + return schema.safeParse(raw); +} + +const valid = { + key: "a", + region: "us-east-1", + url: "postgres://h/db", + replication: { slotName: "s", publicationName: "p", originGeneration: 2 }, +}; + +describe("parseRunOpsShards", () => { + it("returns [] for undefined", () => { + const r = run(undefined); + expect(r.success && r.data).toEqual([]); + }); + it("returns [] for an empty array literal", () => { + const r = run("[]"); + expect(r.success && r.data).toEqual([]); + }); + it("parses a valid single descriptor", () => { + const r = run(JSON.stringify([valid])); + expect(r.success).toBe(true); + if (r.success) expect(r.data[0].key).toBe("a"); + }); + it("fails on malformed JSON", () => { + expect(run("{not json").success).toBe(false); + }); + it("fails on a multi-char key", () => { + expect(run(JSON.stringify([{ ...valid, key: "ab" }])).success).toBe(false); + }); + it("fails on duplicate keys", () => { + const b = { + ...valid, + replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 }, + }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails on duplicate origin generations", () => { + const b = { ...valid, key: "b", url: "postgres://h/b" }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails when both url and aliasOf are set", () => { + expect( + run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])) + .success + ).toBe(false); + }); + it("accepts aliasOf without url or replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", aliasOf: "new" }])).success).toBe(true); + }); + it("fails on an origin generation below 2 or above 255", () => { + const mk = (g: number) => + run( + JSON.stringify([ + { ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }, + ]) + ); + expect(mk(1).success).toBe(false); + expect(mk(256).success).toBe(false); + }); + it("fails when a non-aliased descriptor omits replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe( + false + ); + }); +}); + +describe("validateShardListAgainstNewUrl", () => { + it("passes when the list is empty and no new url", () => { + expect(validateShardListAgainstNewUrl([], undefined)).toBe(true); + }); + it("passes when the list is non-empty and new url is set", () => { + expect(validateShardListAgainstNewUrl([valid as never], "postgres://h/new")).toBe(true); + }); + it("fails when the list is non-empty and new url is unset", () => { + expect(validateShardListAgainstNewUrl([valid as never], undefined)).toBe(false); + }); +}); diff --git a/apps/webapp/test/runStoreShardWiring.test.ts b/apps/webapp/test/runStoreShardWiring.test.ts new file mode 100644 index 00000000000..728f8ee5623 --- /dev/null +++ b/apps/webapp/test/runStoreShardWiring.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { RoutingRunStore } from "@internal/run-store"; +import { buildRunStore } from "~/v3/runStore.server"; + +// Construction-only: buildRunStore wraps clients but never connects, so stub handles suffice. This +// asserts the wiring shape (compat router vs N-way router), not query behaviour. +const stub = () => ({}) as any; + +const baseSplit = { + splitEnabled: true as const, + newWriter: stub(), + newReplica: stub(), + legacyWriter: stub(), + legacyReplica: stub(), + singleWriter: stub(), + singleReplica: stub(), +}; + +describe("buildRunStore shard wiring", () => { + it("split ON with no shards builds the two-store compat router", () => { + const store = buildRunStore(baseSplit); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split ON with two shard descriptors builds the N-way router", () => { + const store = buildRunStore({ + ...baseSplit, + shards: [ + { key: "a", writer: stub(), replica: stub() }, + { key: "b", writer: stub(), replica: stub() }, + ], + }); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split OFF builds the single-store passthrough (not a router)", () => { + const store = buildRunStore({ + splitEnabled: false, + singleWriter: stub(), + singleReplica: stub(), + }); + expect(store).not.toBeInstanceOf(RoutingRunStore); + }); +}); diff --git a/apps/webapp/test/transactionResilience.test.ts b/apps/webapp/test/transactionResilience.test.ts new file mode 100644 index 00000000000..a033f3be985 --- /dev/null +++ b/apps/webapp/test/transactionResilience.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { resolveTransactionResilience } from "~/v3/transactionResilience.server"; + +describe("resolveTransactionResilience per-shard", () => { + it("builds a distinct budget per call, so one shard's storm cannot drain another's", () => { + const a = resolveTransactionResilience("run-ops-shard-a", {}); + const b = resolveTransactionResilience("run-ops-shard-b", {}); + expect(a.startRetry.budget).not.toBe(b.startRetry.budget); + }); + + it("accepts an arbitrary pool label and honours a maxWait override", () => { + expect(() => + resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }) + ).not.toThrow(); + expect(resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }).maxWait).toBe(1234); + }); +}); diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts index b5ea7a51971..8d818ae60bc 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.test.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts @@ -20,6 +20,7 @@ import { generateRunOpsId, generateRunOpsIdV2, generateWaitpointId, + isValidShardChar, parseRunId, parseRunOpsIdBody, parseRunOpsIdV2Body, @@ -418,6 +419,21 @@ describe("parseRunId — v2 arm", () => { }); }); +describe("isValidShardChar", () => { + it("accepts a single [a-z0-9] char", () => { + expect(isValidShardChar("a")).toBe(true); + expect(isValidShardChar("0")).toBe(true); + expect(isValidShardChar("w")).toBe(true); + }); + it("rejects multi-char, empty, uppercase, and punctuation", () => { + expect(isValidShardChar("")).toBe(false); + expect(isValidShardChar("ab")).toBe(false); + expect(isValidShardChar("A")).toBe(false); + expect(isValidShardChar("-")).toBe(false); + expect(isValidShardChar("legacy")).toBe(false); + }); +}); + describe("waitpoint ids: run-ops format with version char w", () => { it("mints a 26-char body per type, with the type char at index 24 and version w at 25", () => { const cases: Array<[WaitpointIdType, string]> = [ diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index 2f436b93a3e..416660ce446 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -40,6 +40,11 @@ export const DEFAULT_REGION_CHAR = "0"; const REGION_CHAR_PATTERN = /^[a-z0-9]$/; // Same slot, same range: the gen-2 shard key is a region char's positional twin. const SHARD_CHAR_PATTERN = REGION_CHAR_PATTERN; +/** True iff `value` is a single valid gen-2 shard char. The descriptor validator and + * `resolveShard` share this so a configured key and a decoded key cannot drift. */ +export function isValidShardChar(value: string): boolean { + return SHARD_CHAR_PATTERN.test(value); +} /** One lowercase [a-z0-9] char per supported region, at RUN_OPS_ID_REGION_INDEX. */ export const REGION_CODES: Readonly> = { "us-east-1": "e", From 38e78f8c7e7c09fdcfd6cd97fd016baa2debc444 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 12:57:46 +0200 Subject: [PATCH 19/28] feat(webapp): deployment lifecycle telemetry events (#4778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployments currently leave little analytical trace. This PR makes every deployment emit two analytics events to enable useful queries. It also enables comparing deployments across build paths, CLI versions, runtimes, and orgs. ### Where the events come from ``` trigger deploy │ ▼ initialize ─────────────────────────────▶ ✨ deployment.initialized │ createdAt ▼ PENDING waiting for a build slot ┐ │ startedAt │ queue time ▼ ┘ INSTALLING build server installs deps ┐ │ installedAt (native paths only) │ install time ▼ ┘ BUILDING the image is built ┐ │ builtAt │ building time ▼ ┘ DEPLOYING indexing + registry push ┐ │ deployedAt / failedAt / canceledAt │ deploying time ▼ ┘ DEPLOYED · FAILED · TIMED_OUT · CANCELED │ └───────────────────────────────────▶ ✨ deployment.finished ``` `deployment.finished` fires exactly once, whichever way the deployment ends, and is backdated to cover the deployment's real lifetime. Not every path visits every state (Depot deploys skip PENDING/INSTALLING, for example) — a phase duration is simply omitted when its state was never entered. ### What each event carries - **Which path built it**: `depot`, `native`, or `native_local_bundle` - **How it ended**: status, plus an error class and message when it failed - **How long each phase took**: queue, install, building, deploying, and total — derived from the timestamps above - **Who and with what**: org, project, environment, runtime, CLI version, and how the deploy was triggered (CLI, GitHub, Vercel) With that, one query gives failure rate per build path, duration percentiles per phase, adoption per CLI version, or a per-org health table. ### Fixes that ride along - The old `deployment.outcome` span was silently dropped ~95% of the time (it was subject to trace sampling). The new events opt out of sampling explicitly, so every deployment is counted. - The fail/timeout/finalize transitions were racy: a late timeout could overwrite a successful deployment. They now use guarded writes, so exactly one caller wins the terminal transition — and exactly one event is emitted. - Canceled deployments previously recorded nothing; they do now. - The deployment's CLI version is now stored at initialization (new nullable column), so even deploys that fail early are attributable to a CLI release. - Telemetry is flushed on shutdown (the last batch used to be lost on every webapp deploy), and an optional second exporter can mirror just these events into a dedicated dataset. --- .changeset/violet-buses-tease.md | 5 + apps/webapp/app/env.server.ts | 4 + apps/webapp/app/routes/api.v1.deployments.ts | 12 +- apps/webapp/app/v3/deploymentTelemetry.ts | 116 +++++++++++ ...eateDeploymentBackgroundWorkerV4.server.ts | 30 +-- .../app/v3/services/deployment.server.ts | 63 ++++-- .../app/v3/services/failDeployment.server.ts | 48 ++++- .../v3/services/finalizeDeployment.server.ts | 45 +++-- .../services/initializeDeployment.server.ts | 16 +- .../recordDeploymentFinished.server.ts | 183 ++++++++++++++++++ .../recordDeploymentOutcome.server.ts | 50 ----- .../v3/services/timeoutDeployment.server.ts | 44 ++++- apps/webapp/app/v3/tracer.server.ts | 71 ++++++- apps/webapp/test/deploymentTelemetry.test.ts | 83 ++++++++ .../migration.sql | 1 + .../database/prisma/schema.prisma | 2 + packages/cli-v3/src/apiClient.ts | 1 + 17 files changed, 661 insertions(+), 113 deletions(-) create mode 100644 .changeset/violet-buses-tease.md create mode 100644 apps/webapp/app/v3/deploymentTelemetry.ts create mode 100644 apps/webapp/app/v3/services/recordDeploymentFinished.server.ts delete mode 100644 apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts create mode 100644 apps/webapp/test/deploymentTelemetry.test.ts create mode 100644 internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql diff --git a/.changeset/violet-buses-tease.md b/.changeset/violet-buses-tease.md new file mode 100644 index 00000000000..191cff77683 --- /dev/null +++ b/.changeset/violet-buses-tease.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the CLI version header on all API requests so deployments are attributable to a CLI version diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c1155d377d3..2b1fba86980 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -940,6 +940,10 @@ const EnvironmentSchema = z DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false), INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + + // Second trace exporter receiving only `deployment.*` spans; they still flow to the main one + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 5be291bae27..184fa996d97 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); try { - const result = await service.call(authenticatedEnv, body.data); + const result = await service.call(authenticatedEnv, body.data, { + cliVersion: parseCliVersionHeader(request), + }); const { deployment, imageRef } = result; const responseBody: InitializeDeploymentResponseBody = { @@ -75,6 +77,14 @@ export async function action({ request, params }: ActionFunctionArgs) { } } +// Client-controlled and persisted, so cap what we accept +const CLI_VERSION_MAX_LENGTH = 128; + +function parseCliVersionHeader(request: Request): string | undefined { + const value = request.headers.get("x-trigger-cli-version"); + return value && value.length <= CLI_VERSION_MAX_LENGTH ? value : undefined; +} + export const loader = createLoaderApiRoute( { searchParams: ApiDeploymentListSearchParams, diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts new file mode 100644 index 00000000000..2dd0725f840 --- /dev/null +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -0,0 +1,116 @@ +import { BuildServerMetadata } from "@trigger.dev/core/v3"; + +/** + * Attribute names for the `deployment.finished` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentFinished.server.ts). + * This module is the single owner of these names — external queries, + * dashboards, and monitors reference them, so treat renames as breaking. + * + * Query gotchas: dedup with `arg_max(_time, *) by deployment.id` (job retries + * can double-emit); the span's `_time` is the deployment's createdAt, so a + * TIMED_OUT event lands backdated by up to the full deploy timeout — monitor + * windows must exceed it; phase durations are omitted (not zero) when a + * boundary timestamp is missing, and `total_ms` excludes local-bundle's + * pre-init client work (esbuild + upload) until the CLI reports timings. + */ +export const DeploymentTelemetryAttributes = { + ORG_ID: "$trigger.org.id", + PROJECT_ID: "$trigger.project.id", + // Project external ref ("proj_…") + PROJECT_REF: "$trigger.project.ref", + ENV_ID: "$trigger.env.id", + // PRODUCTION / STAGING / PREVIEW / DEVELOPMENT + ENV_TYPE: "$trigger.env.type", + // Deployment friendly id — the dedup key + DEPLOYMENT_ID: "deployment.id", + VERSION: "deployment.version", + // finished: terminal status; initialized: initial status (PENDING/BUILDING) + STATUS: "deployment.status", + // status === DEPLOYED; CANCELED is excluded from failure rates + SUCCESS: "deployment.success", + // depot / native / native_local_bundle (see deriveBuildPath) + BUILD_PATH: "deployment.build_path", + // V1 / MANAGED (run engine) + WORKER_TYPE: "deployment.worker_type", + RUNTIME: "deployment.runtime", + // Set at indexing; null for pre-index failures + RUNTIME_VERSION: "deployment.runtime_version", + // From x-trigger-cli-version at init; null for pre-column history + CLI_VERSION: "deployment.cli_version", + TRIGGERED_VIA: "deployment.triggered_via", + COMMIT_SHA: "deployment.commit_sha", + // error.* only on FAILED/TIMED_OUT; CANCELED uses canceled_reason + ERROR_NAME: "deployment.error.name", + ERROR_MESSAGE: "deployment.error.message", + CANCELED_REASON: "deployment.canceled_reason", + // createdAt → terminal (also the span's own duration) + DURATION_TOTAL_MS: "deployment.duration.total_ms", + // createdAt → startedAt; ≈0 when created directly in BUILDING (depot) + DURATION_QUEUE_MS: "deployment.duration.queue_ms", + // startedAt → installedAt; build-server paths only (depot never sets it) + DURATION_INSTALL_MS: "deployment.duration.install_ms", + // (installedAt ?? startedAt) → builtAt + DURATION_BUILDING_MS: "deployment.duration.building_ms", + // builtAt → terminal; for depot dominated by the server-side registry push + DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", +} as const; + +export type DeploymentBuildPath = "native_local_bundle" | "native" | "depot"; + +/** + * Everything that is not a native-build-server deployment falls into the depot + * bucket, including rare `--local-build` deploys (their flag is not persisted). + * `externalBuildData` is NOT a usable depot signal: init writes a placeholder + * for every path. + */ +export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { + const metadata = BuildServerMetadata.safeParse(buildServerMetadata); + + if (metadata.success && metadata.data.isNativeBuild) { + return metadata.data.fromBundle ? "native_local_bundle" : "native"; + } + + return "depot"; +} + +export type DeploymentTimestamps = { + createdAt: Date; + startedAt?: Date | null; + installedAt?: Date | null; + builtAt?: Date | null; +}; + +export type DeploymentDurations = { + totalMs: number; + queueMs?: number; + installMs?: number; + buildingMs?: number; + deployingMs?: number; +}; + +/** + * Timestamp chains are path-shaped (e.g. depot never sets installedAt), so + * each phase is derived only when both of its boundary timestamps exist and + * are ordered. + */ +export function deriveDeploymentDurations( + timestamps: DeploymentTimestamps, + terminalAt: Date +): DeploymentDurations { + const { createdAt, startedAt, installedAt, builtAt } = timestamps; + const buildingFrom = installedAt ?? startedAt; + + return { + totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0), + queueMs: msBetween(createdAt, startedAt), + installMs: msBetween(startedAt, installedAt), + buildingMs: msBetween(buildingFrom, builtAt), + deployingMs: msBetween(builtAt, terminalAt), + }; +} + +function msBetween(from?: Date | null, to?: Date | null): number | undefined { + if (!from || !to) return undefined; + const ms = to.getTime() - from.getTime(); + return ms >= 0 ? ms : undefined; +} diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index d09707a0e83..7215c09c37c 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -18,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -298,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { error: Error, environment: AuthenticatedEnvironment ) { + const failedAt = new Date(); + const errorData = { + name: error.name, + message: error.message, + }; + // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING // transition in `call()`. With idempotent retries, two attempts can run side-by-side; // without the predicate, one attempt's failure could downgrade the deployment after @@ -309,11 +315,8 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { }, data: { status: "FAILED", - failedAt: new Date(), - errorData: { - name: error.name, - message: error.message, - }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); @@ -332,13 +335,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentOutcome({ + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, + deployment: { ...deployment, status: "FAILED", failedAt, errorData }, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, reason: error.message, }); } diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 7a891ae4f61..e8ac53cc475 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -9,6 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -195,10 +196,8 @@ export class DeploymentService extends BaseService { friendlyId: string, data?: Partial> ) { - const validateDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } + const validateDeployment = >( + deployment: T ) => { if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { logger.warn("Attempted cancelling deployment in a final state", { @@ -210,11 +209,7 @@ export class DeploymentService extends BaseService { return okAsync(deployment); }; - const cancelDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } - ) => + const cancelDeployment = >(deployment: T) => fromPromise( this._prisma.workerDeployment.updateMany({ where: { @@ -250,7 +245,25 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) - .andThen(({ deployment }) => + .andTee(({ deployment }) => + recordDeploymentFinished({ + status: "CANCELED", + deployment: { + ...deployment, + status: "CANCELED", + canceledAt: new Date(), + canceledReason: data?.canceledReason ?? null, + }, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.project.id, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environment.id, + environmentType: deployment.environment.type, + }, + }) + ) + .andTee(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { type: "finalized", @@ -259,14 +272,11 @@ export class DeploymentService extends BaseService { message: data?.canceledReason ?? undefined, }, }, - ]) - .orElse((error) => { - logger.error("Failed to append event to deployment event log", { error }); - return okAsync(deployment); - }) - .map(() => deployment) + ]).orTee((error) => { + logger.error("Failed to append event to deployment event log", { error }); + }) ) - .andThen(deleteTimeout) + .andThen(({ deployment }) => deleteTimeout(deployment)) .map(() => undefined); } @@ -484,6 +494,23 @@ export class DeploymentService extends BaseService { select: { status: true, id: true, + friendlyId: true, + version: true, + type: true, + createdAt: true, + startedAt: true, + installedAt: true, + builtAt: true, + deployedAt: true, + failedAt: true, + canceledAt: true, + canceledReason: true, + errorData: true, + runtime: true, + runtimeVersion: true, + cliVersion: true, + triggeredVia: true, + commitSHA: true, buildServerMetadata: true, imageReference: true, shortCode: true, @@ -491,6 +518,8 @@ export class DeploymentService extends BaseService { include: { project: { select: { + id: true, + organizationId: true, externalRef: true, }, }, diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index cb5c622b7b2..534158308cb 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,11 +1,11 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -41,25 +41,53 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + + // Guarded: a concurrent terminal transition can win after the check above + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: { notIn: boundedIn(FINAL_DEPLOYMENT_STATUSES) }, }, data: { status: "FAILED", - failedAt: new Date(), + failedAt, errorData: params.error, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment reached a final state concurrently, skipping fail", { + id: deployment.id, + friendlyId, + }); + return; + } + + // Re-read: the row can gain phase timestamps between the read and the guarded write + const failedDeployment = await this._prisma.workerDeployment.findFirst({ + where: { id: deployment.id }, + }); + + if (!failedDeployment) { + logger.error("Worker deployment disappeared after fail transition", { + id: deployment.id, + friendlyId, + }); + return; + } + + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: failedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, reason: params.error.message, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 51f5b1e37c4..3ee7a1bebf0 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -10,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -66,28 +66,51 @@ export class FinalizeDeploymentService extends BaseService { } const imageDigest = validatedImageDigest(body.imageDigest); + const deployedAt = new Date(); + const imageReference = imageDigest + ? `${deployment.imageReference}@${imageDigest}` + : deployment.imageReference; - // Link the deployment with the background worker - const finalizedDeployment = await this._prisma.workerDeployment.update({ + // Guarded: stops a concurrent transition (e.g. a late timeout) from double-committing + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: "DEPLOYING", }, data: { status: "DEPLOYED", - deployedAt: new Date(), + deployedAt, // Only add the digest, if any - imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + imageReference: imageDigest ? imageReference : undefined, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment left DEPLOYING concurrently, skipping finalize", { + id: deployment.id, + }); + throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); + } + + const finalizedDeployment = { + ...deployment, + status: "DEPLOYED" as const, + deployedAt, + imageReference, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "DEPLOYED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: finalizedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, }); const deploymentService = new DeploymentService(); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index ee55d8bd8d6..56cf50e6879 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -16,6 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; +import { recordDeploymentInitialized } from "./recordDeploymentFinished.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, @@ -56,7 +57,8 @@ export type InitializeDeploymentResult = export class InitializeDeploymentService extends BaseService { public async call( environment: AuthenticatedEnvironment, - payload: InitializeDeploymentRequestBody + payload: InitializeDeploymentRequestBody, + options?: { cliVersion?: string } ): Promise { return this.traceWithEnv("call", environment, async (span) => { if (payload.externalId) { @@ -386,12 +388,24 @@ export class InitializeDeploymentService extends BaseService { commitSHA: payload.gitMeta?.commitSha ?? undefined, externalId: payload.externalId, runtime: payload.runtime ?? environment.project.defaultRuntime ?? undefined, + cliVersion: options?.cliVersion, triggeredVia: payload.triggeredVia ?? undefined, startedAt: initialStatus === "BUILDING" ? new Date() : undefined, }; } ); + recordDeploymentInitialized({ + deployment, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, + }); + const timeoutMs = deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS; diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts new file mode 100644 index 00000000000..8d6727cac7f --- /dev/null +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -0,0 +1,183 @@ +import { ROOT_CONTEXT, SpanStatusCode } from "@opentelemetry/api"; +import { type WorkerDeployment, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; +import { SEMINTATTRS_FORCE_RECORDING, tracer } from "~/v3/tracer.server"; +import { + DeploymentTelemetryAttributes as ATTRS, + deriveBuildPath, + deriveDeploymentDurations, +} from "~/v3/deploymentTelemetry"; + +type TerminalDeploymentStatus = Extract< + WorkerDeploymentStatus, + "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" +>; + +type FinishedDeployment = Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "startedAt" + | "installedAt" + | "builtAt" + | "deployedAt" + | "failedAt" + | "canceledAt" + | "canceledReason" + | "errorData" + | "runtime" + | "runtimeVersion" + | "cliVersion" + | "triggeredVia" + | "commitSHA" +> & { buildServerMetadata: unknown }; + +type EnvironmentInfo = { + organizationId?: string; + projectId?: string; + projectRef?: string; + environmentId?: string; + environmentType?: string; +}; + +/** + * Records a deployment's terminal transition as a single wide + * `deployment.finished` span, backdated createdAt → terminal (attribute + * contract in ../deploymentTelemetry.ts). Call exactly once, only after a + * guarded status write confirmed this caller won the transition. Emitted on + * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never + * throws. + */ +export function recordDeploymentFinished(params: { + status: TerminalDeploymentStatus; + deployment: FinishedDeployment; + environment: EnvironmentInfo; + reason?: string; +}): void { + try { + const { status, deployment, environment, reason } = params; + + const isFailure = status === "FAILED" || status === "TIMED_OUT"; + const terminalAt = + deployment.deployedAt ?? deployment.failedAt ?? deployment.canceledAt ?? new Date(); + const durations = deriveDeploymentDurations(deployment, terminalAt); + const errorData = parseErrorData(deployment.errorData); + + const span = tracer.startSpan( + "deployment.finished", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: status, + [ATTRS.SUCCESS]: status === "DEPLOYED", + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.RUNTIME_VERSION]: deployment.runtimeVersion ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, + [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? (reason ?? errorData?.message) : undefined, + [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, + [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, + [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, + [ATTRS.DURATION_INSTALL_MS]: durations.installMs, + [ATTRS.DURATION_BUILDING_MS]: durations.buildingMs, + [ATTRS.DURATION_DEPLOYING_MS]: durations.deployingMs, + }, + }, + ROOT_CONTEXT + ); + + // CANCELED is deliberately not an error: it stays out of failure rates + if (isFailure) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: reason ?? errorData?.message, + }); + } + + span.end(terminalAt); + } catch (error) { + logger.debug("recordDeploymentFinished failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +/** + * Records a deployment's creation as a zero-duration `deployment.initialized` + * event — the funnel counterpart to `deployment.finished` for detecting + * stuck deployments. Never throws. + */ +export function recordDeploymentInitialized(params: { + deployment: Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "runtime" + | "cliVersion" + | "triggeredVia" + > & { buildServerMetadata: unknown }; + environment: EnvironmentInfo; +}): void { + try { + const { deployment, environment } = params; + + const span = tracer.startSpan( + "deployment.initialized", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: deployment.status, + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + }, + }, + ROOT_CONTEXT + ); + + span.end(deployment.createdAt); + } catch (error) { + logger.debug("recordDeploymentInitialized failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +function parseErrorData(errorData: unknown): { name?: string; message?: string } | undefined { + if (!errorData || typeof errorData !== "object") return undefined; + const record = errorData as Record; + return { + name: typeof record.name === "string" ? record.name : undefined, + message: typeof record.message === "string" ? record.message : undefined, + }; +} diff --git a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts b/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts deleted file mode 100644 index e66a7a6a9f9..00000000000 --- a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { SpanStatusCode } from "@opentelemetry/api"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; -import { logger } from "~/services/logger.server"; -import { tracer } from "~/v3/tracer.server"; - -type TerminalDeploymentStatus = Extract< - WorkerDeploymentStatus, - "DEPLOYED" | "FAILED" | "TIMED_OUT" ->; - -/** - * Records a deployment's terminal status as a `deployment.outcome` span so - * deploy success/failure is queryable from traces (no DB read). Call after each - * terminal-status write. Org/project/env are best-effort; never throws. - */ -export function recordDeploymentOutcome(params: { - status: TerminalDeploymentStatus; - deploymentFriendlyId: string; - organizationId?: string; - projectId?: string; - environmentId?: string; - environmentType?: string; - reason?: string; -}): void { - try { - const span = tracer.startSpan("deployment.outcome", { - attributes: { - "$trigger.org.id": params.organizationId, - "$trigger.project.id": params.projectId, - "$trigger.env.id": params.environmentId, - "$trigger.env.type": params.environmentType, - "deployment.outcome.status": params.status, - "deployment.outcome.success": params.status === "DEPLOYED", - "deployment.outcome.deployment_id": params.deploymentFriendlyId, - "deployment.outcome.reason": params.reason, - }, - }); - - if (params.status !== "DEPLOYED") { - span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason }); - } - - span.end(); - } catch (error) { - logger.debug("recordDeploymentOutcome failed", { - deploymentFriendlyId: params.deploymentFriendlyId, - error: error instanceof Error ? error.message : String(error), - }); - } -} diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 5e417a7863b..63d81ba1930 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -5,7 +5,7 @@ import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -38,25 +38,49 @@ export class TimeoutDeploymentService extends BaseService { return; } - const timedOutDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + const errorData = { message: errorMessage, name: "TimeoutError" }; + + // Guarded: keeps the fromStatus check atomic with the write + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: deployment.status, }, data: { status: "TIMED_OUT", - failedAt: new Date(), - errorData: { message: errorMessage, name: "TimeoutError" }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Deployment moved out of the expected state concurrently, skipping timeout", { + id: deployment.id, + fromStatus, + }); + return; + } + + const timedOutDeployment = { + ...deployment, + status: "TIMED_OUT" as const, + failedAt, + errorData, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "TIMED_OUT", - deploymentFriendlyId: deployment.friendlyId, - organizationId: deployment.environment.project.organizationId, - projectId: deployment.environment.projectId, - environmentId: deployment.environmentId, - environmentType: deployment.environment.type, + deployment: timedOutDeployment, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.projectId, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environmentId, + environmentType: deployment.environment.type, + }, reason: errorMessage, }); diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index cbf9a937c03..7047d65d2b9 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -36,7 +36,9 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; import { BatchSpanProcessor, ParentBasedSampler, + type ReadableSpan, type Sampler, + type Span as SdkTraceSpan, SamplingDecision, type SamplingResult, SimpleSpanProcessor, @@ -69,7 +71,7 @@ import { metricsRegister } from "~/metrics.server"; import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; -const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; +export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; export const DATASOURCE_CONTEXT_KEY = createContextKey("trigger.db.datasource"); @@ -89,6 +91,29 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } +// Mirrors name-prefixed spans into a second exporter; they still flow to the main one +class SpanNamePrefixMirrorProcessor implements SpanProcessor { + constructor( + private readonly _inner: SpanProcessor, + private readonly _prefix: string + ) {} + + onStart(span: SdkTraceSpan, parentContext: Context): void { + this._inner.onStart(span, parentContext); + } + onEnd(span: ReadableSpan): void { + if (span.name.startsWith(this._prefix)) { + this._inner.onEnd(span); + } + } + shutdown(): Promise { + return this._inner.shutdown(); + } + forceFlush(): Promise { + return this._inner.forceFlush(); + } +} + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -270,6 +295,30 @@ function setupTelemetry() { } } + if (env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) { + const deploymentEventExporter = new OTLPTraceExporter({ + url: env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL, + timeoutMillis: 15_000, + headers: parseInternalDeploymentEventHeaders() ?? {}, + }); + + spanProcessors.push( + new SpanNamePrefixMirrorProcessor( + new BatchSpanProcessor(deploymentEventExporter, { + maxExportBatchSize: 64, + scheduledDelayMillis: 1000, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }), + "deployment." + ) + ); + + console.log( + `🔦 Tracer: deployment-event exporter enabled to ${env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL}` + ); + } + const ratioSampler = new TraceIdRatioBasedSampler(samplingRate); const provider = new NodeTracerProvider({ @@ -341,6 +390,13 @@ function setupTelemetry() { instrumentations, }); + // Without this flush every shutdown drops the last batch of spans + const flushOnShutdown = () => { + provider.forceFlush().catch(() => {}); + }; + process.once("SIGTERM", flushOnShutdown); + process.once("SIGINT", flushOnShutdown); + return { tracer: provider.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -874,6 +930,19 @@ function parseInternalTraceHeaders(): Record | undefined { } } +function parseInternalDeploymentEventHeaders(): Record | undefined { + try { + return env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS + ? (JSON.parse(env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS) as Record< + string, + string + >) + : undefined; + } catch { + return; + } +} + function parseInternalMetricsHeaders(): Record | undefined { try { return env.INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts new file mode 100644 index 00000000000..9020f0c7102 --- /dev/null +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; + +describe("deriveBuildPath", () => { + it("classifies fromBundle native builds as native_local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("native_local_bundle"); + }); + + it("classifies native builds without fromBundle as native", () => { + expect(deriveBuildPath({ isNativeBuild: true })).toBe("native"); + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: false })).toBe("native"); + }); + + it("classifies everything else as depot", () => { + expect(deriveBuildPath(null)).toBe("depot"); + expect(deriveBuildPath(undefined)).toBe("depot"); + expect(deriveBuildPath({})).toBe("depot"); + expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); + expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); + // fromBundle alone (skewed writer) must not count as native_local_bundle + expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); + expect(deriveBuildPath("garbage")).toBe("depot"); + }); +}); + +describe("deriveDeploymentDurations", () => { + const t = (seconds: number) => new Date(1_700_000_000_000 + seconds * 1000); + + it("derives all phases for the full build-server chain", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(10), installedAt: t(40), builtAt: t(100) }, + t(130) + ); + + expect(durations).toEqual({ + totalMs: 130_000, + queueMs: 10_000, + installMs: 30_000, + buildingMs: 60_000, + deployingMs: 30_000, + }); + }); + + it("omits install and measures building from startedAt when installedAt is missing (depot)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(0), installedAt: null, builtAt: t(90) }, + t(120) + ); + + expect(durations).toEqual({ + totalMs: 120_000, + queueMs: 0, + installMs: undefined, + buildingMs: 90_000, + deployingMs: 30_000, + }); + }); + + it("omits phases whose boundaries are missing (failed before building)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(5), installedAt: null, builtAt: null }, + t(20) + ); + + expect(durations).toEqual({ + totalMs: 20_000, + queueMs: 5_000, + installMs: undefined, + buildingMs: undefined, + deployingMs: undefined, + }); + }); + + it("never returns negative durations on clock skew", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(10), startedAt: t(5), installedAt: null, builtAt: null }, + t(3) + ); + + expect(durations.totalMs).toBe(0); + expect(durations.queueMs).toBeUndefined(); + }); +}); diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql new file mode 100644 index 00000000000..931ea947926 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a77890930b1..a1423f340c0 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,6 +2265,8 @@ model WorkerDeployment { runtime String? runtimeVersion String? + /// CLI version that initiated the deploy, stamped at initialization + cliVersion String? imageReference String? imagePlatform String @default("linux/amd64") diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 8b9fd56eb1c..a7a07cf40eb 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -1016,6 +1016,7 @@ export class CliApiClient { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", "x-trigger-source": this.source, + "x-trigger-cli-version": VERSION, ...this.getBranchHeader(), }; } From 8da393bf332eeca97fc4eb10391b4da2097ef32e Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 14:13:27 +0200 Subject: [PATCH 20/28] feat(webapp): add org slug and project name to deployment telemetry events (#4785) Adds `$trigger.org.slug` and `$trigger.project.name` attributes to the `deployment.finished` / `deployment.initialized` events (follow-up to #4778). --- apps/webapp/app/v3/deploymentTelemetry.ts | 2 ++ .../services/createDeploymentBackgroundWorkerV4.server.ts | 2 ++ apps/webapp/app/v3/services/deployment.server.ts | 6 ++++++ apps/webapp/app/v3/services/failDeployment.server.ts | 2 ++ apps/webapp/app/v3/services/finalizeDeployment.server.ts | 2 ++ apps/webapp/app/v3/services/initializeDeployment.server.ts | 2 ++ .../app/v3/services/recordDeploymentFinished.server.ts | 6 ++++++ apps/webapp/app/v3/services/timeoutDeployment.server.ts | 5 +++++ 8 files changed, 27 insertions(+) diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts index 2dd0725f840..89a0a674a04 100644 --- a/apps/webapp/app/v3/deploymentTelemetry.ts +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -15,7 +15,9 @@ import { BuildServerMetadata } from "@trigger.dev/core/v3"; */ export const DeploymentTelemetryAttributes = { ORG_ID: "$trigger.org.id", + ORG_SLUG: "$trigger.org.slug", PROJECT_ID: "$trigger.project.id", + PROJECT_NAME: "$trigger.project.name", // Project external ref ("proj_…") PROJECT_REF: "$trigger.project.ref", ENV_ID: "$trigger.env.id", diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index 7215c09c37c..bf2c95745f3 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -340,7 +340,9 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { deployment: { ...deployment, status: "FAILED", failedAt, errorData }, environment: { organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, projectId: environment.projectId, + projectName: environment.project.name, projectRef: environment.project.externalRef, environmentId: environment.id, environmentType: environment.type, diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index e8ac53cc475..0eeaa82b3d6 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -256,7 +256,9 @@ export class DeploymentService extends BaseService { }, environment: { organizationId: deployment.environment.project.organizationId, + organizationSlug: deployment.environment.organization.slug, projectId: deployment.environment.project.id, + projectName: deployment.environment.project.name, projectRef: deployment.environment.project.externalRef, environmentId: deployment.environment.id, environmentType: deployment.environment.type, @@ -519,10 +521,14 @@ export class DeploymentService extends BaseService { project: { select: { id: true, + name: true, organizationId: true, externalRef: true, }, }, + organization: { + select: { slug: true }, + }, }, }, }, diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index 534158308cb..b5a6bbeb24c 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -83,7 +83,9 @@ export class FailDeploymentService extends BaseService { deployment: failedDeployment, environment: { organizationId: authenticatedEnv.organizationId, + organizationSlug: authenticatedEnv.organization.slug, projectId: authenticatedEnv.projectId, + projectName: authenticatedEnv.project.name, projectRef: authenticatedEnv.project.externalRef, environmentId: authenticatedEnv.id, environmentType: authenticatedEnv.type, diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 3ee7a1bebf0..c9d6e7f3932 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -106,7 +106,9 @@ export class FinalizeDeploymentService extends BaseService { deployment: finalizedDeployment, environment: { organizationId: authenticatedEnv.organizationId, + organizationSlug: authenticatedEnv.organization.slug, projectId: authenticatedEnv.projectId, + projectName: authenticatedEnv.project.name, projectRef: authenticatedEnv.project.externalRef, environmentId: authenticatedEnv.id, environmentType: authenticatedEnv.type, diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 56cf50e6879..ca7ce966006 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -399,7 +399,9 @@ export class InitializeDeploymentService extends BaseService { deployment, environment: { organizationId: environment.organizationId, + organizationSlug: environment.organization.slug, projectId: environment.projectId, + projectName: environment.project.name, projectRef: environment.project.externalRef, environmentId: environment.id, environmentType: environment.type, diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts index 8d6727cac7f..4bea471269c 100644 --- a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -37,7 +37,9 @@ type FinishedDeployment = Pick< type EnvironmentInfo = { organizationId?: string; + organizationSlug?: string; projectId?: string; + projectName?: string; projectRef?: string; environmentId?: string; environmentType?: string; @@ -73,7 +75,9 @@ export function recordDeploymentFinished(params: { attributes: { [SEMINTATTRS_FORCE_RECORDING]: true, [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.ORG_SLUG]: environment.organizationSlug, [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_NAME]: environment.projectName, [ATTRS.PROJECT_REF]: environment.projectRef, [ATTRS.ENV_ID]: environment.environmentId, [ATTRS.ENV_TYPE]: environment.environmentType, @@ -147,7 +151,9 @@ export function recordDeploymentInitialized(params: { attributes: { [SEMINTATTRS_FORCE_RECORDING]: true, [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.ORG_SLUG]: environment.organizationSlug, [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_NAME]: environment.projectName, [ATTRS.PROJECT_REF]: environment.projectRef, [ATTRS.ENV_ID]: environment.environmentId, [ATTRS.ENV_TYPE]: environment.environmentType, diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 63d81ba1930..cc935cc4712 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -17,6 +17,9 @@ export class TimeoutDeploymentService extends BaseService { environment: { include: { project: true, + organization: { + select: { slug: true }, + }, }, }, }, @@ -76,7 +79,9 @@ export class TimeoutDeploymentService extends BaseService { deployment: timedOutDeployment, environment: { organizationId: deployment.environment.project.organizationId, + organizationSlug: deployment.environment.organization.slug, projectId: deployment.environment.projectId, + projectName: deployment.environment.project.name, projectRef: deployment.environment.project.externalRef, environmentId: deployment.environmentId, environmentType: deployment.environment.type, From 1801b0e80b2051a0ffdd5f8c5976506b4db27c5e Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:50:20 +0100 Subject: [PATCH 21/28] feat(webapp,docker): run-ops boot interlocks and migrations at N databases (#4780) ## Summary The run-ops boot interlocks and the migration entrypoint each assume exactly two run-ops databases. This generalizes them to any number, so a deployment that configures `RUN_OPS_SHARDS` gets the same safety guarantees it gets today with two stores: no two stores may point at one database, every store that owns its own database must replicate to ClickHouse, and every store must have its schema migrated. With `RUN_OPS_SHARDS` unset, nothing changes. The distinctness check over a two-element set is the pairwise compare it replaces, replication coverage is the check it was, and the entrypoint runs the same two migration invocations. A shard may declare `aliasOf: "new"`, which shares an existing store's client by reference. An aliased shard is not its own database, so it is exempt from the distinctness check and needs no replication slot of its own. Every check keys that exemption on the declared field, never on client object identity: two client objects can sit over one database, which identity comparison cannot see. ## Design **Distinctness.** `probeDistinctDatabases` compared two URLs. It now delegates to `probeDistinctStores`, which reads every fingerprint in parallel and groups them by system identifier and database name. Any two stores under one key refuse the boot. The old pairwise entry point stays, so its existing container tests are the proof that set uniqueness over one pair gives the verdict it gave before. Fail-closed is unchanged: a probe that cannot answer returns not-distinct, because "distinct" is a positive claim a failed probe cannot support. **Co-residency.** The advisory runs once per store against the control plane. The legacy emission keeps its exact call shape and its untagged metric series, so an existing dashboard does not change. Each shard emits its own point carrying its shard key. Every store emits before any enforcement throw, so one offending store never costs another store its metric. **Replication.** `buildReplicationSources` appends one source per shard that owns its own database, taking the slot, publication and origin generation its descriptor declares. `assertReplicationCoversSplit` then requires a source per such shard. That check also closes a hole it inherited. The descriptor parser validates uniqueness among shards only, so a shard could take the slot name, publication name or origin generation of the legacy or the new source. The replication service does validate this, but it throws from its constructor, and the caller reaches that constructor only after shutting the bootstrap instance down: ```ts if (sources.length > 1) { await service.shutdown(); // legacy stream stops here service = new RunsReplicationService({ ... }); // throws: duplicate slotName } ``` The throw was not a `SplitReplicationMisconfiguredError`, so the process stayed up with no replication at all, legacy included, behind one logged line. That is the silent ClickHouse under-count the error exists to prevent. The check now runs at the boot gate, before anything is torn down, and raises a subclass the existing exit path already recognizes. A correct deployment already satisfies it, because two consumers on one WAL slot is a data race that cannot work. **Migrations.** Every shard runs the identical schema, so a new shard is the existing migrations against a new DSN. The runner image has no `jq`, so a small node script prints one DSN per line and the entrypoint loops over them. The loop is a `for` and not a `while read` pipeline: a pipeline subshell swallows a failed migration on any iteration but the last, which would let a broken shard boot. Tracing stays off across the capture and the loop, because `set -x` prints an assignment and a DSN carries credentials. Verified end to end against real Postgres containers for the fingerprint probes, and against the real shell block with a stubbed migration command: an aliased shard is skipped, `directUrl` wins over `url`, a failing shard stops the container on the first failure, and a malformed descriptor stops it before it migrates anything. Stacked on #4764. --------- Co-authored-by: Claude Opus 4.8 --- apps/webapp/app/db.server.ts | 10 + .../runsReplicationInstance.server.ts | 158 ++++++++- ...rolPlaneCoresidencySentinel.server.test.ts | 126 ++++++- .../controlPlaneCoresidencySentinel.server.ts | 93 ++++-- .../distinctDbSentinel.server.ts | 74 +++-- .../v3/runOpsMigration/runOpsSplitReadGate.ts | 37 +++ .../v3/runOpsMigration/splitMode.server.ts | 17 +- apps/webapp/app/v3/runOpsShards.server.ts | 30 ++ apps/webapp/test/runOpsShardDsns.test.ts | 191 +++++++++++ apps/webapp/test/runOpsShards.test.ts | 56 +++- apps/webapp/test/runOpsSplitMode.test.ts | 71 ++++ apps/webapp/test/runOpsSplitReadGate.test.ts | 133 ++++++++ .../test/runsReplicationInstance.test.ts | 312 ++++++++++++++++++ .../distinctDbSentinel.server.test.ts | 115 ++++++- docker/scripts/entrypoint.sh | 38 +++ docker/scripts/runOpsShardDsns.mjs | 132 ++++++++ 16 files changed, 1530 insertions(+), 63 deletions(-) create mode 100644 apps/webapp/test/runOpsShardDsns.test.ts create mode 100644 docker/scripts/runOpsShardDsns.mjs diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 21a9cb9fe22..c63e0aa2d4f 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -599,6 +599,16 @@ export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ controlPlaneReplica: $replica, hasNewUrl: !!env.RUN_OPS_DATABASE_URL, hasLegacyUrl: !!env.RUN_OPS_LEGACY_DATABASE_URL, + // Observability only: a non-distinct shard handle warns and never changes the gen-1 verdict. + // Empty unless RUN_OPS_SHARDS is configured. + shardHandles: runOpsShardHandles.map((handle) => ({ + key: handle.key, + writer: handle.writer, + replica: handle.replica, + // The DECLARED field, not client identity: an aliased shard shares its target's client by + // reference, so identity comparison cannot tell the two apart. + aliasOf: env.RUN_OPS_SHARDS.find((d) => d.key === handle.key)?.aliasOf, + })), logger, }); diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index 164ce07fb92..7c074b1586b 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -3,6 +3,7 @@ import { env } from "~/env.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { singleton } from "~/utils/singleton"; import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server"; +import { nonAliasedShards } from "~/v3/runOpsShards.server"; import { meter, provider } from "~/v3/tracer.server"; import { setRunsReplicationConfiguredSources, @@ -31,6 +32,17 @@ export function buildReplicationSources(args: { newSlotName: string; newPublicationName: string; newOriginGeneration: number; + /** + * Gen-2 shards that own their own database, each with its own slot, publication and origin + * generation. An aliased shard is absent: its target's slot already covers its WAL. + */ + shards?: Array<{ + key: string; + url: string; + /** The DIRECT, non-pooled DSN. Logical replication needs a session-mode connection. */ + directUrl?: string; + replication: { slotName: string; publicationName: string; originGeneration: number }; + }>; }): RunsReplicationSource[] { const legacy: RunsReplicationSource = { id: "legacy", @@ -54,7 +66,29 @@ export function buildReplicationSources(args: { originGeneration: args.newOriginGeneration, }; - return [legacy, next]; + // Shard sources come after the gen-1 pair. Reached only when the new source is on, because + // split is the precondition for a shard to exist at all. The origin generations come from the + // descriptor, which the boot parser already bounds to 2..255 and checks for duplicates; the + // service re-checks uniqueness across every source it is given. + // The DIRECT dsn, not the app writer dsn. A transaction pooler cannot serve the replication + // protocol, and the writer dsn is pooled in a real deployment. Gen-1 keeps the same separation + // through its own RUN_REPLICATION_* variables, and the migration loop prefers directUrl too. + const shardSources: RunsReplicationSource[] = (args.shards ?? []).map((shard) => ({ + id: shardSourceId(shard.key), + pgConnectionUrl: shard.directUrl ?? shard.url, + slotName: shard.replication.slotName, + publicationName: shard.replication.publicationName, + originGeneration: shard.replication.originGeneration, + })); + + return [legacy, next, ...shardSources]; +} + +// The replication source id for a shard. It derives the per-source client name and the key the +// status route probes, so it must be stable and unique across sources. The leader lock is keyed on +// the slot name, not on this id. +function shardSourceId(key: string): string { + return `shard-${key}`; } /** @@ -66,24 +100,106 @@ export function buildReplicationSources(args: { * rather than ship a fleet-wide under-count. */ export class SplitReplicationMisconfiguredError extends Error { - constructor() { + constructor(message?: string) { super( - 'RUN_OPS_SPLIT_ENABLED is on but the runs-replication sources[] has no "new" source: ' + - "run-ops runs on the new DB would not replicate to ClickHouse, under-counting every " + - "ClickHouse-fronted aggregate. Enable the new replication source " + - "(RUN_REPLICATION_NEW_ENABLED / RUN_REPLICATION_RUN_OPS_DATABASE_URL) or turn the split off." + message ?? + 'RUN_OPS_SPLIT_ENABLED is on but the runs-replication sources[] has no "new" source: ' + + "run-ops runs on the new DB would not replicate to ClickHouse, under-counting every " + + "ClickHouse-fronted aggregate. Enable the new replication source " + + "(RUN_REPLICATION_NEW_ENABLED / RUN_REPLICATION_RUN_OPS_DATABASE_URL) or turn the split off." ); this.name = "SplitReplicationMisconfiguredError"; } } +/** + * Two sources that share an identity. The descriptor parser checks uniqueness AMONG shards only, so + * it cannot see the env-configured legacy and new sources: a shard can collide with either. The + * service has its own check, but it throws from the constructor, which the caller reaches only AFTER + * it has shut the bootstrap instance down — leaving the process up with NO replication at all, which + * is the exact silent under-count this family of errors exists to prevent. So the check runs here, + * at the fatal gate, before anything is torn down. + */ +class DuplicateReplicationIdentityError extends SplitReplicationMisconfiguredError { + constructor(field: string, value: unknown) { + super( + `the runs-replication sources[] has two sources with the same ${field} "${String(value)}": ` + + "two consumers on one WAL stream is a data race, and a shared origin generation defeats the " + + "ClickHouse dedup tiebreak. Give every source its own slot, publication and origin generation." + ); + this.name = "DuplicateReplicationIdentityError"; + } +} + +/** + * A configured shard with no replication source of its own. Subclasses the split error on purpose: + * the boot catch site tests `instanceof SplitReplicationMisconfiguredError` to reach + * process.exit(1), and a shard whose runs never reach ClickHouse must take that same exit. + */ +class ShardReplicationMisconfiguredError extends SplitReplicationMisconfiguredError { + constructor(shardKey: string) { + super( + `run-ops shard ${shardKey} is configured but the runs-replication sources[] has no ` + + `"${shardSourceId(shardKey)}" source: runs on that shard would not replicate to ` + + "ClickHouse, under-counting every ClickHouse-fronted aggregate. Give the shard a " + + "replication slot, publication and origin generation, or remove the shard." + ); + this.name = "ShardReplicationMisconfiguredError"; + } +} + +/** + * A shard that replicates but declares no direct dsn. Falling back to its writer dsn is a silent + * trap: if that dsn is pooled, the replication client throws inside start(), which is NOT a + * SplitReplicationMisconfiguredError, so the process stays up with EVERY source down, legacy + * included. Refuse the boot instead. + */ +class ShardDirectUrlMissingError extends SplitReplicationMisconfiguredError { + constructor(shardKey: string) { + super( + `run-ops shard ${shardKey} declares replication but no directUrl: logical replication needs a ` + + "session-mode connection, which a transaction pooler cannot serve. Give the shard a directUrl " + + "pointing at its direct, non-pooled endpoint." + ); + this.name = "ShardDirectUrlMissingError"; + } +} + export function assertReplicationCoversSplit(args: { splitEnabled: boolean; sources: RunsReplicationSource[]; + /** Every configured shard, aliased ones included. An aliased shard needs no source of its own. */ + shards?: Array<{ key: string; aliasOf?: "new"; hasDirectUrl?: boolean }>; }): void { - if (args.splitEnabled && !args.sources.some((s) => s.id === "new")) { + if (!args.splitEnabled) { + return; + } + if (!args.sources.some((s) => s.id === "new")) { throw new SplitReplicationMisconfiguredError(); } + for (const shard of args.shards ?? []) { + // An aliased shard shares its target's database, so the target's slot already carries its WAL. + if (shard.aliasOf !== undefined) continue; + if (!args.sources.some((s) => s.id === shardSourceId(shard.key))) { + throw new ShardReplicationMisconfiguredError(shard.key); + } + if (shard.hasDirectUrl === false) { + throw new ShardDirectUrlMissingError(shard.key); + } + } + + // Cross-source identity, over EVERY source and not only the shards. A correct two-source + // deployment already satisfies this, because two consumers on one WAL slot is a data race that + // cannot work. So this adds a loud failure for a configuration that was already broken silently. + for (const field of ["id", "slotName", "publicationName", "originGeneration"] as const) { + const seen = new Set(); + for (const source of args.sources) { + if (seen.has(source[field])) { + throw new DuplicateReplicationIdentityError(field, source[field]); + } + seen.add(source[field]); + } + } } function initializeRunsReplicationInstance() { @@ -171,6 +287,20 @@ function initializeRunsReplicationInstance() { // The legacy-only instance above is never started in the dual path (no slot/lock // taken). runsReplicationService.server.ts is untouched. The create route also calls // setRunsReplicationGlobal — last-writer-wins is the existing contract. + // An aliased shard replicates through its target's slot, so only the shards that own their own + // database take a source. Coverage is then checked against EVERY descriptor, aliased included. + // The schema requires `replication` on every non-aliased descriptor, so the guard below is a + // type narrowing and not a policy. + const shardReplicationByKey = new Map( + env.RUN_OPS_SHARDS.flatMap((d) => (d.replication ? [[d.key, d.replication] as const] : [])) + ); + const shardsWithReplication = nonAliasedShards(env.RUN_OPS_SHARDS).flatMap((shard) => { + const replication = shardReplicationByKey.get(shard.key); + return replication + ? [{ key: shard.key, url: shard.url, directUrl: shard.directUrl, replication }] + : []; + }); + isSplitEnabled() .then(async (splitEnabled) => { const sources = buildReplicationSources({ @@ -184,10 +314,20 @@ function initializeRunsReplicationInstance() { newSlotName: env.RUN_REPLICATION_NEW_SLOT_NAME, newPublicationName: env.RUN_REPLICATION_NEW_PUBLICATION_NAME, newOriginGeneration: env.RUN_REPLICATION_NEW_ORIGIN_GENERATION, + shards: shardsWithReplication, }); - // Refuse to start replication if split is on but `#new` is not a source. - assertReplicationCoversSplit({ splitEnabled, sources }); + // Refuse to start replication if split is on but `#new` is not a source, or if any shard + // that owns its own database has no source of its own. + assertReplicationCoversSplit({ + splitEnabled, + sources, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + aliasOf: d.aliasOf, + hasDirectUrl: d.directUrl !== undefined, + })), + }); if (sources.length > 1) { // Release the bootstrap instance's eager replication client (Redis + Redlock) diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts index 37301f68743..d08f96529c9 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts @@ -34,6 +34,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: false, probe: async () => ({ coresident: "true" }), emit, @@ -46,6 +47,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { await expect( assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => ({ coresident: "true" }), emit: vi.fn(), @@ -58,6 +60,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => ({ coresident: "unknown", reason: "denied" }), emit, @@ -71,6 +74,7 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const warn = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ ...urls, + shards: [], expectSplit: true, probe: async () => { throw new Error("probe blew up"); @@ -86,9 +90,12 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { const emit = vi.fn(); const probe = vi.fn(); await assertControlPlaneCoresidencyAdvisory({ - legacyUrl: undefined, + // "" and not undefined: ?? only guards nullish, so undefined would read the ambient + // RUN_OPS_LEGACY_DATABASE_URL and this test would depend on the developer's .env. + legacyUrl: "", controlPlaneUrl: "postgres://cp", expectSplit: true, + shards: [], probe, emit, log: noopLog, @@ -97,3 +104,120 @@ describe("assertControlPlaneCoresidencyAdvisory", () => { expect(emit).not.toHaveBeenCalled(); }); }); + +describe("assertControlPlaneCoresidencyAdvisory at N shards", () => { + const urls = { legacyUrl: "postgres://legacy", controlPlaneUrl: "postgres://cp" }; + const shardA = { key: "a", url: "postgres://shard-a" }; + const shardB = { key: "b", url: "postgres://shard-b" }; + + it("emits the legacy verdict with NO shard key, so today's series is unchanged", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(1); + expect(emit).toHaveBeenCalledWith("false"); + }); + + it("emits one tagged verdict per shard, plus the untagged legacy verdict", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [shardA, shardB], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(3); + expect(emit).toHaveBeenCalledWith("false"); + expect(emit).toHaveBeenCalledWith("false", "a"); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("probes each shard against the control plane", async () => { + const probe = vi.fn().mockResolvedValue({ coresident: "false" }); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: false, + shards: [shardA], + probe, + emit: vi.fn(), + log: noopLog, + }); + expect(probe).toHaveBeenCalledWith("postgres://legacy", "postgres://cp", expect.anything()); + expect(probe).toHaveBeenCalledWith("postgres://shard-a", "postgres://cp", expect.anything()); + }); + + it("names the offending shard when enforcement is opted in and a shard is co-resident", async () => { + await expect( + assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA], + probe: async (url: string) => + url === "postgres://shard-a" + ? ({ coresident: "true", reason: "same db" } as const) + : ({ coresident: "false" } as const), + emit: vi.fn(), + log: noopLog, + }) + ).rejects.toThrow(/shard a/i); + }); + + it("emits every store before it throws, so no store loses its metric", async () => { + const emit = vi.fn(); + await expect( + assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA, shardB], + probe: async (url: string) => + url === "postgres://shard-a" + ? ({ coresident: "true", reason: "same db" } as const) + : ({ coresident: "false" } as const), + emit, + log: noopLog, + }) + ).rejects.toThrow(); + expect(emit).toHaveBeenCalledTimes(3); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("degrades one shard's throwing probe to unknown and still reports the others", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + ...urls, + expectSplit: true, + shards: [shardA, shardB], + probe: async (url: string) => { + if (url === "postgres://shard-a") throw new Error("probe blew up"); + return { coresident: "false" } as const; + }, + emit, + log: { info: () => {}, warn: () => {} }, + }); + expect(emit).toHaveBeenCalledWith("unknown", "a"); + expect(emit).toHaveBeenCalledWith("false", "b"); + }); + + it("still probes the shards when there is no legacy DSN", async () => { + const emit = vi.fn(); + await assertControlPlaneCoresidencyAdvisory({ + legacyUrl: "", + controlPlaneUrl: "postgres://cp", + expectSplit: false, + shards: [shardA], + probe: async () => ({ coresident: "false" }), + emit, + log: noopLog, + }); + expect(emit).toHaveBeenCalledTimes(1); + expect(emit).toHaveBeenCalledWith("false", "a"); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts index 1fb741b0e67..870f51c75bc 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.ts @@ -12,6 +12,7 @@ import type { Counter } from "@opentelemetry/api"; import { getMeter } from "@internal/tracing"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { nonAliasedShards, type ShardTarget } from "~/v3/runOpsShards.server"; import { probeControlPlaneCoresidency, type CoresidencyProbeResult, @@ -39,12 +40,14 @@ export type CoresidencyEnforcement = { throw: false } | { throw: true; message: export function resolveCoresidencyEnforcement(args: { coresident: CoresidencyVerdict; expectSplit: boolean; + /** Omitted for the legacy store, so its message stays exactly as it was. */ + shardKey?: string; }): CoresidencyEnforcement { if (args.expectSplit && args.coresident === "true") { + const store = args.shardKey === undefined ? "legacy run-ops DB" : `shard ${args.shardKey}`; return { throw: true, - message: - "RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT is on but the legacy run-ops DB is still co-resident with the control-plane DB; refusing to start.", + message: `RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT is on but the ${store} is still co-resident with the control-plane DB; refusing to start.`, }; } return { throw: false }; @@ -57,46 +60,84 @@ type AdvisoryLogger = { export async function assertControlPlaneCoresidencyAdvisory(deps?: { probe?: typeof probeControlPlaneCoresidency; - emit?: (verdict: CoresidencyVerdict) => void; + /** shardKey is omitted for the legacy store, so its metric series is unchanged at N=0. */ + emit?: (verdict: CoresidencyVerdict, shardKey?: string) => void; log?: AdvisoryLogger; expectSplit?: boolean; legacyUrl?: string; controlPlaneUrl?: string; + shards?: ShardTarget[]; }): Promise { const log = deps?.log ?? logger; const legacyUrl = deps?.legacyUrl ?? env.RUN_OPS_LEGACY_DATABASE_URL; const controlPlaneUrl = deps?.controlPlaneUrl ?? env.CONTROL_PLANE_DATABASE_URL ?? env.DATABASE_URL; - // No legacy DSN (single-DB / self-host) or no control-plane DSN -> nothing to compare. - if (!legacyUrl || !controlPlaneUrl) return; + const shards = deps?.shards ?? nonAliasedShards(env.RUN_OPS_SHARDS); + // No control-plane DSN -> nothing to compare against, for any store. + if (!controlPlaneUrl) return; + + // The legacy store carries NO shard key, so its metric series and its message are unchanged. + // An aliased shard is already absent from `shards`: it shares its target's database on purpose, + // so a co-residency verdict for it would duplicate its target's verdict. + const stores: Array<{ url: string; shardKey?: string }> = [ + ...(legacyUrl ? [{ url: legacyUrl }] : []), + ...shards.map((shard) => ({ url: shard.url, shardKey: shard.key })), + ]; + if (stores.length === 0) return; const probe = deps?.probe ?? probeControlPlaneCoresidency; const emit = deps?.emit ?? - ((verdict: CoresidencyVerdict) => getCoresidentCounter().add(1, { result: verdict })); + ((verdict: CoresidencyVerdict, shardKey?: string) => + getCoresidentCounter().add( + 1, + shardKey === undefined ? { result: verdict } : { result: verdict, shard: shardKey } + )); const expectSplit = deps?.expectSplit ?? env.RUN_OPS_EXPECT_CONTROL_PLANE_SPLIT; - let result: CoresidencyProbeResult; - try { - result = await probe(legacyUrl, controlPlaneUrl, { logger: log }); - } catch (error) { - // Any unexpected throw still degrades to "unknown" — the advisory arm must never crash boot. - log.warn("run-ops control-plane co-residency probe threw; reporting unknown", { error }); - result = { coresident: "unknown", reason: String(error) }; - } + const results = await Promise.all( + stores.map(async (store) => { + let result: CoresidencyProbeResult; + try { + result = await probe(store.url, controlPlaneUrl, { logger: log }); + } catch (error) { + // Any unexpected throw degrades THAT store to "unknown" — the advisory arm must never + // crash boot, and one store's denied probe must not hide another store's verdict. + log.warn("run-ops control-plane co-residency probe threw; reporting unknown", { + error, + shard: store.shardKey, + }); + result = { coresident: "unknown", reason: String(error) }; + } + return { store, result }; + }) + ); - emit(result.coresident); - log.info("run_ops_legacy_control_plane_coresident", { - coresident: result.coresident, - reason: "reason" in result ? result.reason : undefined, - expectSplit, - }); + // Emit and log EVERY store before any enforcement throw, so a failing store never costs + // another store its metric. + for (const { store, result } of results) { + // One argument for the legacy store, so its emission is byte-identical to today's. + if (store.shardKey === undefined) { + emit(result.coresident); + } else { + emit(result.coresident, store.shardKey); + } + log.info("run_ops_legacy_control_plane_coresident", { + coresident: result.coresident, + reason: "reason" in result ? result.reason : undefined, + expectSplit, + shard: store.shardKey, + }); + } - const enforcement = resolveCoresidencyEnforcement({ - coresident: result.coresident, - expectSplit, - }); - if (enforcement.throw) { - throw new Error(enforcement.message); + for (const { store, result } of results) { + const enforcement = resolveCoresidencyEnforcement({ + coresident: result.coresident, + expectSplit, + shardKey: store.shardKey, + }); + if (enforcement.throw) { + throw new Error(enforcement.message); + } } } diff --git a/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts b/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts index 4b2bfd9d986..ed7fb0cb237 100644 --- a/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts @@ -62,32 +62,46 @@ export async function probeControlPlaneCoresidency( } } -export async function probeDistinctDatabases( - legacyUrl: string, - newUrl: string, +export type DistinctTarget = { id: string; url: string }; + +/** + * Set uniqueness over every store that owns its own database. Fail-closed: a probe that cannot + * answer returns NOT distinct, because "distinct" is a positive claim a failed probe cannot support. + * + * Same-cluster-different-database policy (unchanged from the pairwise probe): two databases inside + * the SAME cluster (same system identifier, different current_database()) are reported distinct. + * They are genuinely separate Postgres databases with separate WAL-visible state for our purposes. + * + * An ALIASED shard never appears in `targets`. It shares its target's client by reference, so it is + * not its own database and inclusion would guarantee a duplicate. See nonAliasedShards. + */ +export async function probeDistinctStores( + targets: DistinctTarget[], opts?: { logger?: { warn: (msg: string, meta?: Record) => void } } ): Promise<{ distinct: true } | { distinct: false; reason: string }> { + if (targets.length < 2) { + return { distinct: true }; + } + try { - const [legacy, next] = await Promise.all([ - readDatabaseFingerprint(legacyUrl), - readDatabaseFingerprint(newUrl), - ]); - const sameCluster = legacy.systemIdentifier === next.systemIdentifier; - const sameDb = sameCluster && legacy.databaseName === next.databaseName; - // Same-cluster-different-database policy: two databases inside the SAME cluster - // (same system identifier, different current_database()) are reported distinct: true. - // That is acceptable — they are genuinely separate Postgres databases with separate - // WAL-visible state for our purposes, and the Cloud topology always uses separate - // clusters anyway. A stricter "must be a different cluster" policy would gate on - // sameCluster alone; that is flagged as an open question, not decided here. - if (sameDb) { - const reason = - "run-ops legacy and new URLs resolve to the SAME physical database " + - `(systemIdentifier=${legacy.systemIdentifier}, database=${legacy.databaseName}); ` + - "refusing to enable split — pooler/replica likely."; - opts?.logger?.warn(reason); - return { distinct: false, reason }; + const fingerprints = await Promise.all(targets.map((t) => readDatabaseFingerprint(t.url))); + + const seen = new Map(); + for (const [index, target] of targets.entries()) { + const fingerprint = fingerprints[index]; + const key = `${fingerprint.systemIdentifier}/${fingerprint.databaseName}`; + const first = seen.get(key); + if (first !== undefined) { + const reason = + `run-ops stores "${first}" and "${target.id}" resolve to the SAME physical database ` + + `(systemIdentifier=${fingerprint.systemIdentifier}, database=${fingerprint.databaseName}); ` + + "refusing to enable split — pooler/replica likely."; + opts?.logger?.warn(reason); + return { distinct: false, reason }; + } + seen.set(key, target.id); } + return { distinct: true }; } catch (error) { const reason = `distinct-db sentinel probe failed; failing closed (single-DB). ${String(error)}`; @@ -95,3 +109,19 @@ export async function probeDistinctDatabases( return { distinct: false, reason }; } } + +// The gen-1 pairwise entry point, kept as a thin delegate over a 2-element target list. Set +// uniqueness over one pair IS the pairwise compare, and this function's tests are the proof. +export async function probeDistinctDatabases( + legacyUrl: string, + newUrl: string, + opts?: { logger?: { warn: (msg: string, meta?: Record) => void } } +): Promise<{ distinct: true } | { distinct: false; reason: string }> { + return probeDistinctStores( + [ + { id: "legacy", url: legacyUrl }, + { id: "new", url: newUrl }, + ], + opts + ); +} diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts b/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts index 0872a256508..e70dc29f449 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts @@ -8,6 +8,12 @@ export function computeRunOpsSplitReadEnabled(args: { controlPlaneReplica: unknown; hasNewUrl: boolean; hasLegacyUrl: boolean; + /** + * Gen-2 shard handles. Observability only: a non-distinct shard handle WARNS and never changes the + * returned verdict. A gen-2 fault must not disable the proven gen-1 read fan-out, and the + * distinctness sentinel already fail-closes the boot when two stores share a database. + */ + shardHandles?: Array<{ key: string; writer?: unknown; replica: unknown; aliasOf?: "new" }>; logger?: { warn: (msg: string, meta?: Record) => void }; }): boolean { const newIsDistinctDedicatedClient = @@ -24,5 +30,36 @@ export function computeRunOpsSplitReadEnabled(args: { ); } + // An aliased shard shares its target's client by reference, so identity equality is its correct + // state and never a fault. Keyed on the declared field, not on object identity. + for (const shard of args.shardHandles ?? []) { + if (shard.aliasOf !== undefined) continue; + + // A shard with no replica URL takes its own writer as its replica handle, so its reads go to + // its primary. This is the per-shard analogue of the existing legacy-primary warning. + if (shard.writer !== undefined && shard.replica === shard.writer) { + args.logger?.warn( + `run-ops shard ${shard.key} has no read replica handle; reads for that shard will hit the ` + + "shard primary. Set the shard's replicaUrl to keep replica reads off its primary." + ); + continue; + } + + // Unreachable by construction today: a non-aliased shard always gets a freshly built client. + // Kept as a regression guard, so a future control-plane fallback for shards cannot silently + // route a shard's reads to another database. + if ( + shard.replica === args.controlPlaneWriter || + shard.replica === args.controlPlaneReplica || + shard.replica === args.newReplica + ) { + args.logger?.warn( + `run-ops shard ${shard.key} declares its own database but its replica client is not a ` + + "distinct instance from the control-plane or gen-1 new client; reads for that shard " + + "would not reach its database." + ); + } + } + return enabled; } diff --git a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts index 688f95bac03..b9a4e3dfdf2 100644 --- a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts @@ -6,12 +6,15 @@ */ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; -import { probeDistinctDatabases as defaultProbe } from "./distinctDbSentinel.server"; +import { probeDistinctStores as defaultProbe } from "./distinctDbSentinel.server"; +import { nonAliasedShards, type ShardTarget } from "~/v3/runOpsShards.server"; export type SplitModeConfig = { flagEnabled: boolean; legacyUrl?: string; newUrl?: string; + /** Gen-2 shards that own their own database. Empty (the default) is today's gen-1 pair. */ + shards?: ShardTarget[]; }; export type SplitModeDeps = { @@ -34,9 +37,16 @@ export async function computeSplitEnabled( ); return false; } - // Hard gate #2: runtime sentinel must confirm physically-distinct DBs. + // Hard gate #2: runtime sentinel must confirm physically-distinct DBs. At N stores this is set + // uniqueness over every store that owns its own database, not a compare of the gen-1 pair. An + // aliased shard is already absent from `shards` — it shares its target's client by reference. const probe = deps.probe ?? defaultProbe; - const result = await probe(config.legacyUrl, config.newUrl, { logger: deps.logger }); + const targets = [ + { id: "legacy", url: config.legacyUrl }, + { id: "new", url: config.newUrl }, + ...(config.shards ?? []).map((shard) => ({ id: `shard-${shard.key}`, url: shard.url })), + ]; + const result = await probe(targets, { logger: deps.logger }); return result.distinct === true; } @@ -72,6 +82,7 @@ export function isSplitEnabled(): Promise { flagEnabled: env.RUN_OPS_SPLIT_ENABLED, legacyUrl: env.RUN_OPS_LEGACY_DATABASE_URL, newUrl: env.RUN_OPS_DATABASE_URL, + shards: nonAliasedShards(env.RUN_OPS_SHARDS), }, { logger } ); diff --git a/apps/webapp/app/v3/runOpsShards.server.ts b/apps/webapp/app/v3/runOpsShards.server.ts index 23c45efa404..12cc460aa06 100644 --- a/apps/webapp/app/v3/runOpsShards.server.ts +++ b/apps/webapp/app/v3/runOpsShards.server.ts @@ -122,3 +122,33 @@ export function validateShardListAgainstNewUrl( ): boolean { return shards.length === 0 || !!newUrl; } + +// A shard that owns its own physical database. Every boot check that must not treat two handles +// over one database as two databases derives its target list from here: the distinctness sentinel, +// the coresidency loop, the read gate, the replication sources and the migration loop. +export type ShardTarget = { + key: string; + url: string; + replicaUrl?: string; + directUrl?: string; +}; + +// An aliased shard shares its target's client BY REFERENCE, so it is never its own database. The +// exemption keys on the declared `aliasOf` field, never on client object identity: two store objects +// can sit over one database, which identity comparison cannot see. +export function nonAliasedShards(shards: RunOpsShardDescriptor[]): ShardTarget[] { + const targets: ShardTarget[] = []; + for (const shard of shards) { + if (shard.aliasOf !== undefined) continue; + // Unreachable for a valid descriptor (the schema requires exactly one of url/aliasOf); this is + // the type narrowing, not a second policy. + if (shard.url === undefined) continue; + targets.push({ + key: shard.key, + url: shard.url, + ...(shard.replicaUrl !== undefined ? { replicaUrl: shard.replicaUrl } : {}), + ...(shard.directUrl !== undefined ? { directUrl: shard.directUrl } : {}), + }); + } + return targets; +} diff --git a/apps/webapp/test/runOpsShardDsns.test.ts b/apps/webapp/test/runOpsShardDsns.test.ts new file mode 100644 index 00000000000..a2cef335b1d --- /dev/null +++ b/apps/webapp/test/runOpsShardDsns.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from "vitest"; +// The entrypoint calls this file with plain `node`, so it takes no path alias and no bundler. +import { shardMigrationDsns } from "../../../docker/scripts/runOpsShardDsns.mjs"; + +const shardA = { + key: "a", + region: "us-east-1", + url: "postgres://h/a", + replication: { slotName: "sa", publicationName: "pa", originGeneration: 2 }, +}; + +describe("shardMigrationDsns", () => { + it("returns nothing when the variable is unset", () => { + expect(shardMigrationDsns(undefined)).toEqual([]); + }); + + it("returns nothing when the variable is blank", () => { + expect(shardMigrationDsns(" ")).toEqual([]); + }); + + it("returns nothing for an empty array", () => { + expect(shardMigrationDsns("[]")).toEqual([]); + }); + + it("throws on invalid JSON, so the entrypoint stops before it migrates", () => { + expect(() => shardMigrationDsns("{not json")).toThrow(/not valid JSON/i); + }); + + it("throws when the value is JSON but not an array", () => { + expect(() => shardMigrationDsns('{"key":"a"}')).toThrow(/not a JSON array/i); + }); + + it("returns the url of a shard that owns its own database", () => { + expect(shardMigrationDsns(JSON.stringify([shardA]))).toEqual(["postgres://h/a"]); + }); + + it("prefers directUrl over url, because migrations must not go through a pooler", () => { + const withDirect = { ...shardA, directUrl: "postgres://h/a-direct" }; + expect(shardMigrationDsns(JSON.stringify([withDirect]))).toEqual(["postgres://h/a-direct"]); + }); + + it("skips an aliased shard, because its target's invocation already migrates it", () => { + const aliased = { key: "z", region: "us-east-1", aliasOf: "new" }; + expect(shardMigrationDsns(JSON.stringify([shardA, aliased]))).toEqual(["postgres://h/a"]); + }); + + it("rejects a descriptor with neither url nor aliasOf", () => { + const noUrl = { key: "b", region: "us-east-1" }; + expect(() => shardMigrationDsns(JSON.stringify([shardA, noUrl]))).toThrow(/exactly one/i); + }); + + it("keeps declaration order across several shards", () => { + const shardB = { ...shardA, key: "b", url: "postgres://h/b" }; + expect(shardMigrationDsns(JSON.stringify([shardA, shardB]))).toEqual([ + "postgres://h/a", + "postgres://h/b", + ]); + }); + + it("throws when an entry is not an object", () => { + expect(() => shardMigrationDsns('["postgres://h/a"]')).toThrow(/not an object/i); + }); +}); + +describe("shardMigrationDsns line protocol", () => { + // One DSN per line is the protocol with entrypoint.sh, so a line break would split one DSN into + // two bogus ones. The URL parser strips ASCII line breaks, so nothing upstream rejects this. + it("throws when a DSN holds a line break", () => { + const bad = { ...shardA, url: "postgres://h/a\npostgres://evil/db" }; + expect(() => shardMigrationDsns(JSON.stringify([bad]))).toThrow(/line break/i); + }); + + it("throws when a directUrl holds a carriage return", () => { + const bad = { ...shardA, directUrl: "postgres://h/a\rx" }; + expect(() => shardMigrationDsns(JSON.stringify([bad]))).toThrow(/line break/i); + }); +}); + +// The script and the boot schema validate the same variable, so they must agree. If the script is +// laxer, the entrypoint migrates a database and the application then refuses to start, which breaks +// the fail-before-migration contract the entrypoint exists to hold. +describe("shardMigrationDsns matches the descriptor contract", () => { + it("rejects an aliasOf value the schema does not allow", () => { + const bad = [{ key: "a", region: "r", url: "postgres://h/a", aliasOf: "other" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/aliasOf/i); + }); + + it("rejects a shard that owns its database but declares no replication", () => { + const bad = [{ key: "b", region: "r", url: "postgres://h/b" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/replication/i); + }); + + it("rejects a descriptor that sets both url and aliasOf", () => { + const bad = [{ key: "c", region: "r", url: "postgres://h/c", aliasOf: "new" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/exactly one/i); + }); + + it("rejects a descriptor that sets neither url nor aliasOf", () => { + const bad = [{ key: "d", region: "r" }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/exactly one/i); + }); + + it("still accepts a valid aliased descriptor and skips it", () => { + const ok = [{ key: "z", region: "r", aliasOf: "new" }]; + expect(shardMigrationDsns(JSON.stringify(ok))).toEqual([]); + }); + + it("still accepts a valid owning descriptor", () => { + expect(shardMigrationDsns(JSON.stringify([shardA]))).toEqual(["postgres://h/a"]); + }); +}); + +// The boot schema URL-validates url, replicaUrl and directUrl with isValidDatabaseUrl. The script +// must not be laxer, or a valid descriptor ahead of an invalid one gets its database migrated before +// the configuration is rejected. +describe("shardMigrationDsns validates descriptor values", () => { + const rep = { slotName: "s", publicationName: "p", originGeneration: 2 }; + + it("rejects a url that is not a parseable URL", () => { + const bad = [{ key: "a", region: "r", url: "not a URL", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/url/i); + }); + + it("rejects a directUrl that is not a parseable URL", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", directUrl: "nope", replication: rep }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/directUrl/i); + }); + + it("rejects a replicaUrl that is not a parseable URL", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", replicaUrl: "nope", replication: rep }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/replicaUrl/i); + }); + + it("rejects an empty schema search param, matching the boot schema", () => { + const bad = [{ key: "a", region: "r", url: "postgres://h/a?schema=", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/url/i); + }); + + it("rejects a multi-char shard key", () => { + const bad = [{ key: "ab", region: "r", url: "postgres://h/a", replication: rep }]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/key/i); + }); + + it("rejects an origin generation outside 2..255", () => { + for (const gen of [1, 256]) { + const bad = [ + { + key: "a", + region: "r", + url: "postgres://h/a", + replication: { ...rep, originGeneration: gen }, + }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/originGeneration/i); + } + }); + + it("rejects a replication block with a blank slot name", () => { + const bad = [ + { key: "a", region: "r", url: "postgres://h/a", replication: { ...rep, slotName: "" } }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(bad))).toThrow(/slotName/i); + }); + + // The failure must come BEFORE any DSN is handed back, so no database is migrated first. + it("emits nothing when a later descriptor is invalid", () => { + const mixed = [ + { key: "a", region: "r", url: "postgres://h/a", replication: rep }, + { key: "b", region: "r", url: "not a URL", replication: { ...rep, originGeneration: 3 } }, + ]; + expect(() => shardMigrationDsns(JSON.stringify(mixed))).toThrow(/url/i); + }); + + it("still accepts a fully valid descriptor with all three URLs", () => { + const ok = [ + { + key: "a", + region: "r", + url: "postgres://h/a?schema=public", + replicaUrl: "postgres://h/a-replica?schema=public", + directUrl: "postgres://h/a-direct?schema=public", + replication: rep, + }, + ]; + expect(shardMigrationDsns(JSON.stringify(ok))).toEqual(["postgres://h/a-direct?schema=public"]); + }); +}); diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts index fef7e925e75..c7fd0e0defa 100644 --- a/apps/webapp/test/runOpsShards.test.ts +++ b/apps/webapp/test/runOpsShards.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; import { z } from "zod"; -import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; +import { + nonAliasedShards, + parseRunOpsShards, + validateShardListAgainstNewUrl, + type RunOpsShardDescriptor, +} from "~/v3/runOpsShards.server"; function run(raw: string | undefined) { const schema = z.string().optional().transform(parseRunOpsShards); @@ -82,3 +87,52 @@ describe("validateShardListAgainstNewUrl", () => { expect(validateShardListAgainstNewUrl([valid as never], undefined)).toBe(false); }); }); + +describe("nonAliasedShards", () => { + const shardA: RunOpsShardDescriptor = { + key: "a", + region: "us-east-1", + url: "postgres://h/a", + replication: { slotName: "sa", publicationName: "pa", originGeneration: 2 }, + }; + const shardB: RunOpsShardDescriptor = { + key: "b", + region: "us-west-2", + url: "postgres://h/b", + replicaUrl: "postgres://h/b-replica", + directUrl: "postgres://h/b-direct", + replication: { slotName: "sb", publicationName: "pb", originGeneration: 3 }, + }; + const aliased: RunOpsShardDescriptor = { + key: "z", + region: "us-east-1", + aliasOf: "new", + }; + + it("returns [] for no descriptors", () => { + expect(nonAliasedShards([])).toEqual([]); + }); + + it("keeps a shard that owns its own database", () => { + expect(nonAliasedShards([shardA])).toEqual([{ key: "a", url: "postgres://h/a" }]); + }); + + it("carries the replica and direct URLs when the descriptor sets them", () => { + expect(nonAliasedShards([shardB])).toEqual([ + { + key: "b", + url: "postgres://h/b", + replicaUrl: "postgres://h/b-replica", + directUrl: "postgres://h/b-direct", + }, + ]); + }); + + it("drops an aliased shard, because it shares its target's database", () => { + expect(nonAliasedShards([aliased])).toEqual([]); + }); + + it("keeps declaration order across a mixed list", () => { + expect(nonAliasedShards([shardA, aliased, shardB]).map((s) => s.key)).toEqual(["a", "b"]); + }); +}); diff --git a/apps/webapp/test/runOpsSplitMode.test.ts b/apps/webapp/test/runOpsSplitMode.test.ts index 7ce2bec3a5d..fd7da6f356c 100644 --- a/apps/webapp/test/runOpsSplitMode.test.ts +++ b/apps/webapp/test/runOpsSplitMode.test.ts @@ -61,6 +61,77 @@ describe("computeSplitEnabled (pure)", () => { }); }); +describe("computeSplitEnabled shard targets", () => { + const shardA = { key: "a", url: "postgres://shard-a" }; + const shardB = { key: "b", url: "postgres://shard-b" }; + + it("probes the gen-1 pair only when no shard is configured", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: true }); + await computeSplitEnabled( + { flagEnabled: true, legacyUrl: "postgres://a", newUrl: "postgres://b" }, + { probe } + ); + expect(probe).toHaveBeenCalledWith( + [ + { id: "legacy", url: "postgres://a" }, + { id: "new", url: "postgres://b" }, + ], + expect.anything() + ); + }); + + it("appends one target per shard, keyed by shard id", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: true }); + await computeSplitEnabled( + { + flagEnabled: true, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA, shardB], + }, + { probe } + ); + expect(probe).toHaveBeenCalledWith( + [ + { id: "legacy", url: "postgres://a" }, + { id: "new", url: "postgres://b" }, + { id: "shard-a", url: "postgres://shard-a" }, + { id: "shard-b", url: "postgres://shard-b" }, + ], + expect.anything() + ); + }); + + it("stays single-DB when a shard duplicates another store", async () => { + const probe = vi.fn().mockResolvedValue({ distinct: false, reason: "same DB" }); + expect( + await computeSplitEnabled( + { + flagEnabled: true, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA], + }, + { probe } + ) + ).toBe(false); + }); + + it("never probes a shard when the flag is off", async () => { + const probe = vi.fn(); + await computeSplitEnabled( + { + flagEnabled: false, + legacyUrl: "postgres://a", + newUrl: "postgres://b", + shards: [shardA], + }, + { probe } + ); + expect(probe).not.toHaveBeenCalled(); + }); +}); + describe("assertSplitRealtimeInterlock (pure)", () => { it("throws when split is on but the native realtime backend is off", () => { expect(() => diff --git a/apps/webapp/test/runOpsSplitReadGate.test.ts b/apps/webapp/test/runOpsSplitReadGate.test.ts index 4deb0bb5329..430abbff859 100644 --- a/apps/webapp/test/runOpsSplitReadGate.test.ts +++ b/apps/webapp/test/runOpsSplitReadGate.test.ts @@ -165,3 +165,136 @@ describe("computeRunOpsSplitReadEnabled", () => { }); }); }); + +describe("computeRunOpsSplitReadEnabled shard handles", () => { + const shardA = { __tag: "shard-a" }; + const shardB = { __tag: "shard-b" }; + const base = { + newReplica: dedicatedNew, + controlPlaneWriter: cpWriter, + controlPlaneReplica: cpReplica, + hasNewUrl: true, + hasLegacyUrl: true, + }; + + it("does not warn when every shard handle is a distinct instance", () => { + const warn = vi.fn(); + const enabled = computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "a", replica: shardA }, + { key: "b", replica: shardB }, + ], + logger: { warn }, + }); + expect(enabled).toBe(true); + expect(warn).not.toHaveBeenCalled(); + }); + + it("warns, naming the shard, when a shard replica aliases a control-plane handle", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: cpReplica }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatch(/shard a/i); + }); + + it("warns when a shard replica aliases the gen-1 new replica", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: dedicatedNew }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("does NOT warn for an aliased shard, because sharing is its purpose", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "z", replica: dedicatedNew, aliasOf: "new" as const }], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + // The distinctness sentinel already fail-closes the boot on this condition. A gen-2 fault must + // not disable the proven gen-1 read fan-out on top of that. + it("keeps the gen-1 verdict when a shard handle is not distinct", () => { + expect( + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", replica: cpReplica }], + }) + ).toBe(true); + }); + + // The reachable case. A shard with no replicaUrl gets its own WRITER as its replica handle, so its + // reads go to its primary. selectRunOpsTopology does exactly that (db.server.ts), which makes this + // the per-shard analogue of the existing legacy "reads will hit the legacy primary" warning. + it("warns when a shard has no distinct replica handle, so its reads hit its primary", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardA }], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatch(/shard a/i); + expect(warn.mock.calls[0][0]).toMatch(/primary/i); + }); + + it("does not warn when a shard has its own distinct replica handle", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardB }], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does NOT warn about primary reads for an aliased shard", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "z", writer: dedicatedNew, replica: dedicatedNew, aliasOf: "new" as const }, + ], + logger: { warn }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("keeps the gen-1 verdict when a shard reads from its primary", () => { + expect( + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [{ key: "a", writer: shardA, replica: shardA }], + }) + ).toBe(true); + }); + + it("warns once per offending shard", () => { + const warn = vi.fn(); + computeRunOpsSplitReadEnabled({ + ...base, + shardHandles: [ + { key: "a", replica: cpReplica }, + { key: "b", replica: cpWriter }, + ], + logger: { warn }, + }); + expect(warn).toHaveBeenCalledTimes(2); + }); + + it("is unchanged when no shard handle is supplied", () => { + const warn = vi.fn(); + expect(computeRunOpsSplitReadEnabled({ ...base, logger: { warn } })).toBe(true); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/webapp/test/runsReplicationInstance.test.ts b/apps/webapp/test/runsReplicationInstance.test.ts index 67f595c597c..86aeb86116e 100644 --- a/apps/webapp/test/runsReplicationInstance.test.ts +++ b/apps/webapp/test/runsReplicationInstance.test.ts @@ -202,6 +202,225 @@ describe("assertReplicationCoversSplit (boot gate-coupling)", () => { }); }); +describe("replication sources at N shards", () => { + const baseArgs = { + legacyUrl: "postgres://legacy", + legacySlotName: "task_runs_to_clickhouse_v1", + legacyPublicationName: "task_runs_to_clickhouse_v1_publication", + legacyOriginGeneration: 0, + newSlotName: "task_runs_to_clickhouse_v2", + newPublicationName: "task_runs_to_clickhouse_v2_publication", + newOriginGeneration: 1, + splitEnabled: true, + newUrl: "postgres://new", + }; + + const shardA = { + key: "a", + url: "postgres://shard-a", + replication: { + slotName: "task_runs_to_clickhouse_shard_a", + publicationName: "task_runs_to_clickhouse_shard_a_publication", + originGeneration: 2, + }, + }; + const shardB = { + key: "b", + url: "postgres://shard-b", + replication: { + slotName: "task_runs_to_clickhouse_shard_b", + publicationName: "task_runs_to_clickhouse_shard_b_publication", + originGeneration: 3, + }, + }; + + it("appends nothing when no shard is configured", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(sources.map((s) => s.id)).toEqual(["legacy", "new"]); + }); + + it("appends one source per shard, after legacy and new", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(sources.map((s) => s.id)).toEqual(["legacy", "new", "shard-a", "shard-b"]); + expect(sources[2]).toEqual({ + id: "shard-a", + pgConnectionUrl: "postgres://shard-a", + slotName: "task_runs_to_clickhouse_shard_a", + publicationName: "task_runs_to_clickhouse_shard_a_publication", + originGeneration: 2, + }); + }); + + it("appends no shard source when the new source is off, because split is the precondition", () => { + const sources = buildReplicationSources({ + ...baseArgs, + splitEnabled: false, + shards: [shardA], + }); + expect(sources.map((s) => s.id)).toEqual(["legacy"]); + }); + + it("throws when a shard that owns its database has no source", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }], + }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the uncovered shard in the message", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).toThrow(/shard b/i); + }); + + it("does NOT throw when every shard has its own source", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).not.toThrow(); + }); + + it("does NOT require a source for an aliased shard, because its target's slot covers it", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "z", aliasOf: "new" }], + }) + ).not.toThrow(); + }); + + it("does NOT check shard coverage when split is off", () => { + const sources = buildReplicationSources({ ...baseArgs, splitEnabled: false, shards: [] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: false, + sources, + shards: [{ key: "a" }], + }) + ).not.toThrow(); + }); + + // The catch site keys on `instanceof SplitReplicationMisconfiguredError` to reach + // process.exit(1). A shard with no replication must reach the same exit. + it("raises an error the existing exit path recognizes", () => { + try { + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ ...baseArgs, shards: [] }), + shards: [{ key: "a" }], + }); + expect.unreachable("expected a throw"); + } catch (error) { + expect(error).toBeInstanceOf(SplitReplicationMisconfiguredError); + } + }); + + // F1 class: the descriptor parser checks uniqueness AMONG shards only. It cannot see the + // env-configured legacy and new sources, so a shard can collide with them. The service's own + // check throws too late: the caller has already shut the bootstrap instance down, so the throw + // leaves the process up with NO replication at all. These must fail at the fatal boot gate. + it("throws when a shard's slot name collides with the gen-1 new slot", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { ...shardA, replication: { ...shardA.replication, slotName: baseArgs.newSlotName } }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("throws when a shard's origin generation collides with the gen-1 new generation", () => { + const sources = buildReplicationSources({ + ...baseArgs, + newOriginGeneration: 2, + shards: [shardA], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("throws when a shard's publication name collides with the legacy publication", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { + ...shardA, + replication: { ...shardA.replication, publicationName: baseArgs.legacyPublicationName }, + }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the colliding field in the message", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { ...shardA, replication: { ...shardA.replication, slotName: baseArgs.newSlotName } }, + ], + }); + expect(() => + assertReplicationCoversSplit({ splitEnabled: true, sources, shards: [{ key: "a" }] }) + ).toThrow(/slotName/); + }); + + it("does NOT throw when every shard's slot, publication and generation are its own", () => { + const sources = buildReplicationSources({ ...baseArgs, shards: [shardA, shardB] }); + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources, + shards: [{ key: "a" }, { key: "b" }], + }) + ).not.toThrow(); + }); + + // The service validates sources before it builds a single replication client, so this needs no + // container. RunsReplicationService itself is untouched by this change: the check already exists. + it("rejects two shards that share an origin generation, via the service's own check", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [shardA, { ...shardB, replication: { ...shardB.replication, originGeneration: 2 } }], + }); + + expect( + () => + new RunsReplicationService({ + clickhouseFactory: new TestReplicationClickhouseFactory( + new ClickHouse({ url: "http://127.0.0.1:1", name: "unused", logLevel: "warn" }) + ), + serviceName: "runs-replication", + pgConnectionUrl: "postgres://legacy", + slotName: "unused", + publicationName: "unused", + redisOptions: { host: "127.0.0.1", port: 1 }, + sources, + logLevel: "warn", + }) + ).toThrow(/duplicate originGeneration/i); + }); +}); + describe("RunsReplication new-source backfill origin generation (integration)", () => { replicationContainerTest( "backfill via the new source tags the ClickHouse row with the new origin generation (gen=1), not gen=0", @@ -434,3 +653,96 @@ describe("RunsReplication multi-source wiring (integration)", () => { } ); }); + +// Logical replication needs a session-mode connection, which a transaction pooler cannot serve. +// The app writer DSN is pooled in a real deployment, so a shard replication source must take the +// shard's DIRECT url. Getting this wrong throws inside service.start(), which is not a +// SplitReplicationMisconfiguredError, so the process stays up with every source down. +describe("shard replication uses the direct connection", () => { + const baseArgs = { + legacyUrl: "postgres://legacy", + legacySlotName: "v1", + legacyPublicationName: "v1_pub", + legacyOriginGeneration: 0, + newSlotName: "v2", + newPublicationName: "v2_pub", + newOriginGeneration: 1, + splitEnabled: true, + newUrl: "postgres://new", + }; + const rep = { slotName: "sa", publicationName: "pa", originGeneration: 2 }; + + it("prefers the shard's directUrl over its pooled url", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [ + { + key: "a", + url: "postgres://pooled:6432/shard_a", + directUrl: "postgres://direct:5432/shard_a", + replication: rep, + }, + ], + }); + const shard = sources.find((s) => s.id === "shard-a"); + expect(shard?.pgConnectionUrl).toBe("postgres://direct:5432/shard_a"); + }); + + it("falls back to url when no directUrl is given", () => { + const sources = buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://only-url/shard_a", replication: rep }], + }); + const shard = sources.find((s) => s.id === "shard-a"); + expect(shard?.pgConnectionUrl).toBe("postgres://only-url/shard_a"); + }); + + it("refuses the boot when a replicating shard declares no directUrl", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: false }], + }) + ).toThrow(SplitReplicationMisconfiguredError); + }); + + it("names the shard and the reason in that failure", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: false }], + }) + ).toThrow(/shard a.*directUrl|directUrl.*shard a/is); + }); + + it("does NOT refuse when the replicating shard declares a directUrl", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ + ...baseArgs, + shards: [{ key: "a", url: "postgres://u", directUrl: "postgres://d", replication: rep }], + }), + shards: [{ key: "a", hasDirectUrl: true }], + }) + ).not.toThrow(); + }); + + it("does NOT require a directUrl for an aliased shard", () => { + expect(() => + assertReplicationCoversSplit({ + splitEnabled: true, + sources: buildReplicationSources({ ...baseArgs, shards: [] }), + shards: [{ key: "z", aliasOf: "new", hasDirectUrl: false }], + }) + ).not.toThrow(); + }); +}); diff --git a/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts b/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts index d2baaa6404a..562d50b63d5 100644 --- a/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts @@ -1,7 +1,10 @@ import { heteroPostgresTest } from "@internal/testcontainers"; import { PrismaClient } from "@trigger.dev/database"; import { describe, expect, vi } from "vitest"; -import { probeDistinctDatabases } from "~/v3/runOpsMigration/distinctDbSentinel.server"; +import { + probeDistinctDatabases, + probeDistinctStores, +} from "~/v3/runOpsMigration/distinctDbSentinel.server"; // Spinning up two separate postgres clusters and probing each can exceed the 5s default. vi.setConfig({ testTimeout: 60_000 }); @@ -62,3 +65,113 @@ describe("probeDistinctDatabases", () => { } ); }); + +describe("probeDistinctStores (set uniqueness at N)", () => { + heteroPostgresTest( + "reports distinct for two separate physical clusters", + async ({ uri14, uri17 }) => { + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + ]); + expect(result).toEqual({ distinct: true }); + } + ); + + heteroPostgresTest("reports distinct for a single target", async ({ uri14 }) => { + const result = await probeDistinctStores([{ id: "legacy", url: uri14 }]); + expect(result).toEqual({ distinct: true }); + }); + + heteroPostgresTest("reports distinct for an empty target list", async () => { + const result = await probeDistinctStores([]); + expect(result).toEqual({ distinct: true }); + }); + + heteroPostgresTest( + "reports NOT distinct, naming both ids, when two targets resolve to one database", + async ({ uri14, uri17 }) => { + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: uri14 }, + ]); + expect(result.distinct).toBe(false); + if (result.distinct === false) { + expect(result.reason).toMatch(/same physical database/i); + expect(result.reason).toMatch(/legacy/); + expect(result.reason).toMatch(/shard-a/); + } + } + ); + + // A pairwise implementation that only ever compares the first two targets passes every other + // case in this file and fails this one: legacy vs new is clean, and the duplicate pair is + // shard against shard on a third database. + heteroPostgresTest( + "catches a duplicate between two SHARDS while the gen-1 pair is clean", + async ({ postgresContainer14, uri14, uri17 }) => { + const shardDb = `sentinel_shard_dupe_${Date.now()}`; + const admin = new PrismaClient({ + datasources: { + db: { url: urlWithDatabase(postgresContainer14.getConnectionUri(), "postgres") }, + }, + }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${shardDb}"`); + } finally { + await admin.$disconnect(); + } + const shardUrl = urlWithDatabase(uri14, shardDb); + + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: shardUrl }, + { id: "shard-b", url: shardUrl }, + ]); + expect(result.distinct).toBe(false); + if (result.distinct === false) { + expect(result.reason).toMatch(/shard-a/); + expect(result.reason).toMatch(/shard-b/); + } + } + ); + + heteroPostgresTest( + "reports distinct for two databases in the SAME cluster", + async ({ postgresContainer14, uri14, uri17 }) => { + const otherDb = `sentinel_set_other_${Date.now()}`; + const admin = new PrismaClient({ + datasources: { + db: { url: urlWithDatabase(postgresContainer14.getConnectionUri(), "postgres") }, + }, + }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${otherDb}"`); + } finally { + await admin.$disconnect(); + } + + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: urlWithDatabase(uri14, otherDb) }, + ]); + expect(result).toEqual({ distinct: true }); + } + ); + + heteroPostgresTest( + "fails closed to NOT distinct when one target cannot be reached", + async ({ uri14, uri17 }) => { + const unreachable = "postgresql://nobody:nobody@127.0.0.1:1/does_not_exist"; + const result = await probeDistinctStores([ + { id: "legacy", url: uri14 }, + { id: "new", url: uri17 }, + { id: "shard-a", url: unreachable }, + ]); + expect(result.distinct).toBe(false); + } + ); +}); diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 1e5c7c7cab0..58d17bc6ce9 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -49,6 +49,44 @@ else echo "RUN_OPS_LEGACY_DIRECT_URL not set, skipping legacy run-ops migrations." fi +# Run-ops shards: migrate every gen-2 shard that owns its own database. Each shard runs the +# identical schema, so this is the existing run-ops migrations against a new DSN. An aliased shard is +# skipped by the DSN script: it IS its target's database. Installs that never set RUN_OPS_SHARDS +# skip this entirely. +{ set +x; } 2>/dev/null +if [ -n "$RUN_OPS_SHARDS" ]; then + set -x + if [ "$SKIP_RUN_OPS_SHARD_MIGRATIONS" != "1" ]; then + echo "Running run-ops shard migrations" + # Tracing stays OFF from here to the end of the loop: `set -x` prints an assignment, so + # capturing a DSN under tracing would put the credentials in the logs. + { set +x; } 2>/dev/null + # A malformed descriptor exits 1 here, so the container stops before it migrates anything. + shard_dsns=$(node scripts/runOpsShardDsns.mjs) + # A `for` loop and NOT `... | while read`: a pipeline subshell would swallow a failed migration + # on any iteration but the last. Here `set -e` stops the boot on the first shard that fails. + # The whole loop runs in a subshell, so the IFS and `set -f` changes need no restore and cannot + # leak into the rest of the entrypoint. IFS is newline-only so a DSN is never split on other + # whitespace, and `set -f` stops a DSN query string (it holds `?`) from acting as a glob. + ( + IFS=' +' + set -f + for shard_dsn in $shard_dsns; do + # Tracing stays off so `set -x` never prints the DSN (with credentials) to the logs. + RUN_OPS_DATABASE_URL="$shard_dsn" DIRECT_URL="$shard_dsn" pnpm --filter @internal/run-ops-database db:migrate:deploy + done + ) + set -x + echo "Run-ops shard migrations done" + else + echo "SKIP_RUN_OPS_SHARD_MIGRATIONS=1, skipping run-ops shard migrations." + fi +else + set -x + echo "RUN_OPS_SHARDS not set, skipping run-ops shard migrations." +fi + if [ "$SKIP_DASHBOARD_AGENT_MIGRATIONS" != "1" ]; then echo "Running dashboard agent migrations" pnpm --filter @internal/dashboard-agent-db db:migrate:deploy diff --git a/docker/scripts/runOpsShardDsns.mjs b/docker/scripts/runOpsShardDsns.mjs new file mode 100644 index 00000000000..dc47d1500f7 --- /dev/null +++ b/docker/scripts/runOpsShardDsns.mjs @@ -0,0 +1,132 @@ +// Print the migration DSN of every run-ops shard that owns its own database, one per line, so +// entrypoint.sh can loop over them. The runner image has no `jq`, and this script is unit-tested, +// which an inline `node -e` string could not be. +// +// Contract: +// RUN_OPS_SHARDS unset or blank -> print nothing, exit 0 (single-DB and gen-1-only installs) +// invalid JSON, or not an array -> message on stderr, exit 1 (the app rejects the same value) +// a descriptor with `aliasOf` -> skipped; it shares its target's database +// the DSN -> `directUrl` if set, else `url`; skipped if neither is set +// +// Never print a DSN to stderr or to a log: stdout is consumed by the caller, nothing else. + +// Mirrors isValidDatabaseUrl in the webapp: parseable by URL(), and no empty `schema` param. +function assertDatabaseUrl(value, field, key) { + try { + const parsed = new URL(value); + if (parsed.searchParams.get("schema") === "") { + throw new Error("empty schema param"); + } + } catch { + throw new Error(`RUN_OPS_SHARDS[${key}]: ${field} is not a valid database URL`); + } +} + +/** + * Mirrors the boot schema's rules for one descriptor. Kept deliberately narrow: it checks the rules + * that make the application REJECT the value at boot, so an invalid descriptor stops the entrypoint + * before any migration runs. It does NOT reject unknown fields, because doing so would fail the + * entrypoint on a descriptor a newer application accepts, which is drift in the other direction. + */ +function assertDescriptor(d) { + const key = typeof d.key === "string" ? d.key : "?"; + if (typeof d.key !== "string" || !/^[a-z0-9]$/.test(d.key)) { + throw new Error(`RUN_OPS_SHARDS[${key}]: key must be a single [a-z0-9] char`); + } + if (typeof d.region !== "string" || d.region === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: region is required`); + } + + const hasAlias = d.aliasOf !== undefined && d.aliasOf !== null; + if (hasAlias && d.aliasOf !== "new") { + throw new Error(`RUN_OPS_SHARDS[${key}]: aliasOf must be "new", got "${d.aliasOf}"`); + } + + const hasUrl = d.url !== undefined && d.url !== null; + if (hasUrl === hasAlias) { + throw new Error(`RUN_OPS_SHARDS[${key}]: exactly one of url or aliasOf is required`); + } + + for (const field of ["url", "replicaUrl", "directUrl"]) { + const value = d[field]; + if (value === undefined || value === null) continue; + if (typeof value !== "string" || value === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: ${field} is not a valid database URL`); + } + assertDatabaseUrl(value, field, key); + } + + if (hasAlias) return; + + const rep = d.replication; + if (rep === undefined || rep === null || typeof rep !== "object") { + throw new Error(`RUN_OPS_SHARDS[${key}]: replication is required unless aliasOf is set`); + } + for (const field of ["slotName", "publicationName"]) { + if (typeof rep[field] !== "string" || rep[field] === "") { + throw new Error(`RUN_OPS_SHARDS[${key}]: replication.${field} must be a non-empty string`); + } + } + const gen = rep.originGeneration; + if (!Number.isInteger(gen) || gen < 2 || gen > 255) { + throw new Error( + `RUN_OPS_SHARDS[${key}]: replication.originGeneration must be an integer 2..255` + ); + } +} + +export function shardMigrationDsns(raw) { + if (raw === undefined || raw === null || String(raw).trim() === "") { + return []; + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("RUN_OPS_SHARDS is not valid JSON"); + } + + if (!Array.isArray(parsed)) { + throw new Error("RUN_OPS_SHARDS is not a JSON array"); + } + + // Validate EVERY descriptor before collecting any DSN, so a valid descriptor ahead of an invalid + // one never gets its database migrated before the configuration is rejected. + for (const descriptor of parsed) { + if (descriptor === null || typeof descriptor !== "object" || Array.isArray(descriptor)) { + throw new Error("RUN_OPS_SHARDS holds an entry that is not an object"); + } + assertDescriptor(descriptor); + } + + const dsns = []; + for (const descriptor of parsed) { + // An aliased shard is the same database as its target, which its own invocation migrates. + if (descriptor.aliasOf !== undefined && descriptor.aliasOf !== null) continue; + + const dsn = descriptor.directUrl ?? descriptor.url; + // One DSN per line IS the protocol with the caller, so a DSN holding a line break would split + // into two bogus DSNs. The URL parser strips ASCII line breaks, so nothing else rejects it. + if (/[\r\n]/.test(dsn)) { + throw new Error("RUN_OPS_SHARDS holds a DSN containing a line break"); + } + dsns.push(dsn); + } + return dsns; +} + +// `import.meta.main` is not available on every supported node, so compare argv instead. +const invokedDirectly = + process.argv[1] !== undefined && process.argv[1].endsWith("runOpsShardDsns.mjs"); + +if (invokedDirectly) { + try { + for (const dsn of shardMigrationDsns(process.env.RUN_OPS_SHARDS)) { + process.stdout.write(`${dsn}\n`); + } + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + } +} From 02e6157d12078647fbf686b1394c38bfc269c702 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:20:19 +0100 Subject: [PATCH 22/28] feat(run-store): add an execution-snapshot store decorator behind an off-by-default dial (#4765) ## Summary Adds a `RunStore` decorator that mirrors execution snapshots into Redis alongside Postgres, plus the orphan-key sweep and the fault-injection suite that prove the write protocol converges after a crash. Nothing constructs it, so merging this changes no behaviour: the configuration, the production wiring and the Redis client all arrive in later work. The execution-state log is the hottest table in the run graph, and moving it out of Postgres has to happen without a big-bang cutover. This is the attachment point for that: a decorator that wraps the existing storage interface and intercepts only the methods that touch snapshots, so none of the many callers change. ## Design Write order is the correctness property, and the two orders differ on purpose. A transition writes Postgres first and Redis second. A crash in the gap leaves a run whose latest snapshot is stale, which is the state the heartbeat stall watchdog already heals in production today. A birth writes Redis first and Postgres second. A crash there leaves an unreachable key for a run that does not exist. Postgres first would instead leave a run with no snapshot at all, which the engine treats as a hard error, so the run would be stuck. Each order is chosen so the state a crash leaves behind is the harmless one. A lost cross-store write is never recovered by a transaction or an outbox; recovery is always the existing stall and repair job. A failed append retries, then hands the run to that job, and never rethrows, because Postgres has already committed and a throw would turn a healable gap into a caller-visible error. Inside a transaction the Redis half is staged and flushed only after the commit, so a rollback cannot leave Redis holding a transition that never happened. Reads are shape matched. Two of the snapshot reads take arbitrary Prisma arguments, and a key-value store cannot answer an arbitrary query, so the decorator recognises exactly the shapes the engine sends and delegates everything else. A miss falls back to Postgres, which is also how runs created before any cutover keep working. The sweep reaps under two rules, because neither can see what the other leaves behind. A finished run whose keyspace never received its completion expiry gets one applied. A keyspace with no run row at all, past an age threshold, is deleted; that is a crashed birth, which is non-terminal so it carries no expiry and has no run row, so the first rule can never match it. ## Inertness Three independent reasons this is a no-op if merged alone: - Nothing constructs the decorator or the Redis store outside tests. - No configuration reaches it, so the dial stays at its off position, which is a pass-through that makes no Redis call. - The existing Postgres store gains an off-by-default flag and two optional input fields. Both default to today's behaviour, and only the decorator would ever supply them. ## Notes for review The snapshot id and the creation instant are both minted by the decorator and written into both stores, so one snapshot has one identity and one timestamp wherever it is read. Without that, the two stores disagree on values that later tooling has to compare, and the cursor for a snapshot window resolved from one store misfilters the window walked in the other. Three defects in this work passed the full existing test suites before being found by review rather than by a test: the decorator wrote no wait cycle at all, the snapshot window dropped the ordering used to give each completed waitpoint its position in a batch, and the two stores stamped different creation times. The common cause was that no test drove a snapshot that actually carried waitpoints, and that the parity suite compared a timestamp against a value it had just read back from the row it was checking. Both gaps now have tests. --- internal-packages/redis/src/index.ts | 21 +- .../engine/tests/finalRunStatusParity.test.ts | 20 + .../engine/tests/helpers/decoratedStore.ts | 75 ++ .../engine/tests/snapshotStoreChaos.test.ts | 402 +++++++ .../tests/snapshotStoreReadGate.test.ts | 304 ++++++ .../src/PostgresRunStore.snapshotId.test.ts | 150 +++ ...ostgresRunStore.snapshotTimestamps.test.ts | 132 +++ .../PostgresRunStore.snapshotWrites.test.ts | 312 ++++++ .../run-store/src/PostgresRunStore.ts | 306 ++++-- .../src/delegatingRunStore.forwarding.test.ts | 188 ++++ .../run-store/src/delegatingRunStore.test.ts | 118 +++ .../run-store/src/delegatingRunStore.ts | 750 ++++++++++++++ internal-packages/run-store/src/index.ts | 5 + .../redisSnapshotStore.sinceCreatedAt.test.ts | 193 ++++ .../run-store/src/redisSnapshotStore.ts | 383 ++++++- .../run-store/src/runStoreMethodNames.ts | 115 ++ .../src/snapshotEntry.parity.test.ts | 522 ++++++++++ .../run-store/src/snapshotEntry.test.ts | 185 ++++ .../run-store/src/snapshotEntry.ts | 168 +++ .../run-store/src/snapshotFaultInjection.ts | 45 + .../src/snapshotOrphanSweeper.cluster.test.ts | 160 +++ .../src/snapshotOrphanSweeper.confirm.test.ts | 360 +++++++ .../src/snapshotOrphanSweeper.test.ts | 469 +++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 567 ++++++++++ .../run-store/src/snapshotReadShapes.test.ts | 151 +++ .../run-store/src/snapshotReadShapes.ts | 95 ++ ...skRunExecutionSnapshotStore.births.test.ts | 285 +++++ .../taskRunExecutionSnapshotStore.off.test.ts | 100 ++ ...nExecutionSnapshotStore.readCohort.test.ts | 104 ++ ...askRunExecutionSnapshotStore.reads.test.ts | 420 ++++++++ ...unExecutionSnapshotStore.redisOnly.test.ts | 321 ++++++ ...kRunExecutionSnapshotStore.staging.test.ts | 286 +++++ ...ExecutionSnapshotStore.transitions.test.ts | 454 ++++++++ .../src/taskRunExecutionSnapshotStore.ts | 980 ++++++++++++++++++ ...utionSnapshotStore.waitpointCycles.test.ts | 706 +++++++++++++ .../src/testFixtures/snapshotIdFixture.ts | 167 +++ internal-packages/run-store/src/types.ts | 42 + 37 files changed, 9918 insertions(+), 143 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.test.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.ts create mode 100644 internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts create mode 100644 internal-packages/run-store/src/runStoreMethodNames.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.parity.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.ts create mode 100644 internal-packages/run-store/src/snapshotFaultInjection.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.ts create mode 100644 internal-packages/run-store/src/snapshotReadShapes.test.ts create mode 100644 internal-packages/run-store/src/snapshotReadShapes.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts create mode 100644 internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..622efe613fc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,24 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { type Cluster, Redis, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** + * Either endpoint shape. A component that only issues key-addressed commands works against both, so + * it should accept this rather than pin itself to a standalone connection. Commands with no key — + * SCAN above all — do NOT fan out across a cluster, so anything that issues one must iterate + * `cluster.nodes("master")` itself. + */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache diff --git a/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts new file mode 100644 index 00000000000..50eba9d3a04 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts @@ -0,0 +1,20 @@ +// The snapshot sweeper needs to know which run statuses are terminal, and it cannot import that +// list: run-engine depends on run-store, not the other way round. So the list is duplicated, and +// this is the only thing that keeps the copy honest. +// +// Without it, a status added here and not there makes the sweeper treat a finished run as live and +// never apply its completion expiry. A status removed here and not there makes it treat a live run +// as finished. The second one reaps state a run is still using. +import { describe, expect, it } from "vitest"; +import { FINAL_RUN_STATUSES } from "@internal/run-store"; +import { getFinalRunStatuses } from "../statuses.js"; + +describe("terminal run statuses", () => { + it("match between the engine and the snapshot sweeper", () => { + expect([...FINAL_RUN_STATUSES].sort()).toEqual([...getFinalRunStatuses()].sort()); + }); + + it("are not empty, so the comparison cannot pass vacuously", () => { + expect(FINAL_RUN_STATUSES.length).toBeGreaterThan(0); + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..ea3447acc56 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,402 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..17c186d074d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,304 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts new file mode 100644 index 00000000000..8026f2f99ec --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts @@ -0,0 +1,132 @@ +// The caller's instant must land in BOTH timestamp columns, on BOTH schema variants. +// +// `updatedAt` is declared `@updatedAt`, which Prisma manages itself, so whether an explicit value +// survives a create is a property of the client rather than of the schema. The two variants are +// separately generated clients over separately declared schemas, so agreeing declarations are not +// evidence that they agree in behaviour. This asserts it on each. +// +// It matters because the decorator writes one instant to both stores. If Prisma overrode it here, +// Postgres and Redis would hold different values for a column the comparator checks for equality, +// on every snapshot. +import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +/** Five minutes in the past, so a column default could never coincide with it. */ +const STAMP = new Date(Date.now() - 5 * 60 * 1000); + +async function writeSnapshot( + prisma: AnyClient, + schemaVariant: "legacy" | "dedicated", + suffix: string +) { + const scope = + schemaVariant === "dedicated" + ? { + environmentId: `env_${suffix}`, + projectId: `proj_${suffix}`, + organizationId: `org_${suffix}`, + } + : await seedLegacyScope(prisma as PrismaClient, suffix); + + const store = new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant, + }); + + const runId = generateInternalId(); + const id = generateInternalId(); + + await (prisma as PrismaClient).taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${suffix}`, + runtimeEnvironmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + organizationId: scope.organizationId, + projectId: scope.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + } as never, + }); + + await store.createExecutionSnapshot({ + id, + createdAt: STAMP, + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + projectId: scope.projectId, + organizationId: scope.organizationId, + }); + + return (prisma as PrismaClient).taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); +} + +async function seedLegacyScope(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + environmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + }; +} + +describe("snapshot timestamps are the caller's, on both schema variants", () => { + heteroPostgresTest("legacy client honours the supplied instant", async ({ prisma14 }) => { + const row = await writeSnapshot(prisma14, "legacy", "tsleg"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + // The one Prisma manages. If it overrode the value, the two stores would disagree here on + // every snapshot. + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + }); + + heteroRunOpsPostgresTest( + "dedicated client honours the supplied instant", + async ({ prisma17 }) => { + const row = await writeSnapshot(prisma17, "dedicated", "tsded"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..dda0f483bf5 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,312 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 33604d01148..7af040eb99c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -118,6 +118,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +645,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +658,22 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput): + | { + executionSnapshots: { + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; + }; + } + | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -726,6 +750,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -743,7 +769,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +781,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +798,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,23 +837,26 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -923,20 +950,21 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1129,18 +1157,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1260,42 +1289,46 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1363,18 +1396,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1435,21 +1469,21 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1969,6 +2003,8 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, + createdAt, run, snapshot, previousSnapshotId, @@ -1984,10 +2020,65 @@ export class PostgresRunStore implements RunStore { error, } = input; + // Left possibly-undefined ON PURPOSE. Prisma omits an undefined key, so the column keeps taking + // whatever it took before this method was touched: the schema declares `completedWaitpointOrder + // String[]` with no default and the column is nullable, so an omitted key stores NULL, not `{}`. + // Defaulting here would send `{}` instead and change what a live write stores. + // + // The redis-only echo below DOES need a concrete array, because it returns the row shape to the + // caller and that field is not nullable in the payload type. That default belongs to the echo, + // not to the write, so the two are kept apart. + const completedWaitpointOrder = completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id); + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = createdAt ?? new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder: completedWaitpointOrder ?? [], + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, @@ -2007,10 +2098,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, diff --git a/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts new file mode 100644 index 00000000000..081e9efe1d4 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts @@ -0,0 +1,188 @@ +// Every declared parameter must actually reach the delegate. +// +// The compiler cannot check this. A forwarder that omits a trailing OPTIONAL argument compiles +// cleanly, and the effect is silent: `findLatestExecutionSnapshot` would stop applying its tenant +// scope, and `upsertWaitpointTag` would stop applying its residency hint, so a write would land on +// the wrong database. Both of those shipped in this file before this test existed. +// +// So this reads the source of the base against the source of the interface and asserts that each +// forward passes exactly the parameters its signature declares, in order. Source-level, because +// that is the only place the property is visible. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const dir = join(import.meta.dirname); +const interfaceSource = readFileSync(join(dir, "types.ts"), "utf8"); +const baseSource = readFileSync(join(dir, "delegatingRunStore.ts"), "utf8"); + +/** Replaces comments and string bodies with spaces, so neither can shift a brace depth. */ +function blank(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) j += text[j] === "\\" ? 2 : 1; + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +function interfaceBody(source: string): string { + const blanked = blank(source); + const decl = "export interface RunStore {"; + const start = blanked.indexOf(decl) + decl.length; + let depth = 1; + let end = start; + while (depth > 0 && end < blanked.length) { + if (blanked[end] === "{") depth += 1; + else if (blanked[end] === "}") depth -= 1; + if (depth > 0) end += 1; + } + return blanked.slice(start, end); +} + +/** Splits a balanced parameter list on top-level commas. */ +function splitParams(signature: string): string[] { + // A generic member reads `name(...)`, so the parameter list starts after the + // balanced angle block, not at the first parenthesis. + let searchFrom = 0; + const angle = signature.indexOf("<"); + const paren = signature.indexOf("("); + if (angle !== -1 && angle < paren) { + let angleDepth = 0; + for (let i = angle; i < signature.length; i++) { + if (signature[i] === "<") angleDepth += 1; + else if (signature[i] === ">") { + angleDepth -= 1; + if (angleDepth === 0) { + searchFrom = i; + break; + } + } + } + } + + const open = signature.indexOf("(", searchFrom); + let depth = 0; + let close = open; + for (let i = open; i < signature.length; i++) { + if ("([{<".includes(signature[i]!)) depth += 1; + else if (")]}>".includes(signature[i]!)) { + depth -= 1; + if (depth === 0) { + close = i; + break; + } + } + } + const inner = signature.slice(open + 1, close); + const parts: string[] = []; + let level = 0; + let current = ""; + for (const ch of inner) { + if ("([{<".includes(ch)) level += 1; + else if (")]}>".includes(ch)) level -= 1; + if (ch === "," && level === 0) { + parts.push(current); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) parts.push(current); + return parts; +} + +function paramNames(signature: string): string[] { + return splitParams(signature) + .map((p) => /^\s*([A-Za-z_$][\w$]*)\s*\??\s*:/.exec(p)?.[1]) + .filter((n): n is string => Boolean(n)); +} + +/** Member name to its declared parameter names, for members with a single signature. */ +function declaredParams(): Map { + const body = interfaceBody(interfaceSource); + const spans: string[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < body.length; i++) { + const ch = body[i]!; + if ("{([".includes(ch)) level += 1; + else if ("})]".includes(ch)) level -= 1; + else if (ch === ";" && level === 0) { + spans.push(body.slice(from, i)); + from = i + 1; + } + } + + const seen = new Map(); + for (const span of spans) { + const match = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(span); + if (!match) continue; + const name = match[1]!; + seen.set(name, [...(seen.get(name) ?? []), paramNames(span)]); + } + + // Overloaded members forward through a cast and apply the whole argument list, so they are not + // subject to this check. + return new Map( + [...seen].filter(([, sigs]) => sigs.length === 1).map(([n, sigs]) => [n, sigs[0]!]) + ); +} + +describe("the pass-through forwards every declared parameter", () => { + const declared = declaredParams(); + + it("parsed the interface, so a parse failure cannot pass this suite", () => { + expect(declared.size).toBeGreaterThan(50); + expect(declared.get("expireParkedRun")).toEqual(["runId", "data", "tx"]); + expect(declared.get("findLatestExecutionSnapshot")).toEqual([ + "runId", + "client", + "environmentId", + ]); + }); + + it("passes exactly the declared parameters, in order, for every single-signature member", () => { + const wrong: string[] = []; + + for (const [name, params] of declared) { + const forward = new RegExp(`return this\\.delegate\\.${name}\\(([^;]*)\\);`).exec(baseSource); + + if (!forward) { + wrong.push(`${name}: no forward found`); + continue; + } + + const passed = forward[1]! + .split(",") + .map((a) => a.trim()) + .filter(Boolean); + + if (passed.join(",") !== params.join(",")) { + wrong.push(`${name}: declares (${params.join(", ")}) but forwards (${passed.join(", ")})`); + } + } + + expect(wrong).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7fdcd9ac07b --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,118 @@ +// What this suite covers, and why it is now small. +// +// `DelegatingRunStore` restates every interface signature and forwards its arguments by name, so +// most of what a pass-through can get wrong is a compile error rather than a test failure: +// +// member of the interface missing -> `implements RunStore`, TS2420 +// public member the interface lacks -> the parity assertion in the base +// forwarded to the wrong delegate member -> argument types do not match, TS2345/TS2322 +// arguments reordered -> same +// +// Three things remain invisible to the compiler, and they are what is left here. +// +// First, the seven overloaded members. TypeScript cannot express one body that satisfies an overload +// set, so their single implementation forwards through a cast, and the cast is exactly where a +// wrong-member forward would stop being a type error. +// +// Second, a dropped OPTIONAL argument. Omitting a trailing `tx` compiles cleanly and silently stops +// forwarding the caller's transaction. +// +// Third, whether a data property is read live or captured once at construction. Both typecheck; only +// one is correct. +// +// No database is involved in whether a pass-through passes through, so none is started. Behaviour +// against a real store is covered by the container suites for the decorator built on this base. +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +/** + * The members whose implementation forwards through a cast, because they are overloaded. These are + * the only methods where the compiler is not already checking the forward. + */ +const OVERLOADED_MEMBERS = [ + "finalizeRun", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", +] as const; + +type ProbedCall = { name: string; args: unknown[] }; + +/** + * Records what was called and answers with a per-member sentinel, so a forward to the wrong member + * returns the wrong value rather than merely returning something. + */ +function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { + const calls: ProbedCall[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + if ((RUN_STORE_PROPERTY_NAMES as readonly string[]).includes(prop)) { + return `property:${prop}`; + } + return (...args: unknown[]) => { + calls.push({ name: prop, args }); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every method to the member of the same name", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]()).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + }); + + it("covers every overloaded member, so the list cannot rot", () => { + // If a member gains or loses overloads, the cast set changes and this suite should follow. + for (const name of OVERLOADED_MEMBERS) { + expect(RUN_STORE_METHOD_NAMES).toContain(name); + } + }); + + it("forwards arguments untouched through an overloaded member's cast", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + const args = ["first", { second: true }, undefined, 4]; + + for (const name of OVERLOADED_MEMBERS) { + base[name](...args); + } + + // The overloaded implementations apply the whole argument list, so every argument survives, + // including a trailing optional the typed members would legitimately drop. + for (const call of calls) { + expect(call.args).toEqual(args); + } + expect(calls.map((c) => c.name)).toEqual([...OVERLOADED_MEMBERS]); + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..c7fe3225c76 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,750 @@ +// A pass-through over another RunStore. +// +// It exists so a decorator can override the handful of methods it cares about and inherit the rest. +// +// Every member restates the interface signature and forwards its arguments BY NAME, so the +// forwarding is itself type-checked: a body that called the wrong delegate method, or dropped an +// argument, does not compile. That is the whole point of the shape. An untyped forwarder would let +// both mistakes through, because a pass-through has no other behaviour to catch them. +// +// Seven members are overloaded. Their overloads are declared so callers keep the full contract, and +// their single implementation signature is the one place a cast appears: TypeScript cannot express +// one body that satisfies an overload set without it. +// +// Keeping this in step with the interface is not a matter of memory. `implements RunStore` rejects a +// member that is missing, and the assertion at the foot of the file rejects one the interface never +// declared. + +import type { + BatchTaskRun, + BatchTaskRunItemStatus, + Prisma, + PrismaClientOrTransaction, + TaskRun, + TaskRunStatus, + WaitpointTag, +} from "@trigger.dev/database"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { Residency } from "@trigger.dev/core/v3/isomorphic"; +import type { + ClearIdempotencyKeyInput, + CompletionSnapshotInput, + CreateBatchTaskRunData, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateFailedRunInput, + CreateRunInput, + ExpireSnapshotInput, + FinalizeRunData, + ForWaitpointCompletionContext, + IdempotencyKeyRunMatch, + LockRunData, + PromotePendingVersionArgs, + ReadClient, + RescheduleSnapshotInput, + RewriteDebouncedRunData, + RunStore, + TaskRunWithWaitpoint, + WaitpointColocationOptions, +} from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + return this.delegate.runInTransaction(runId, fn); + } + + createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise { + return this.delegate.createRun(params, tx); + } + + createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createCancelledRun(params, tx); + } + + createFailedRun( + params: CreateFailedRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createFailedRun(params, tx); + } + + startAttempt( + runId: string, + data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.startAttempt(runId, data, args, tx); + } + + completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + recordRetryOutcome( + runId: string, + data: { machinePreset?: string; usageDurationMs: number; costInCents: number }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.recordRetryOutcome(runId, data, args, tx); + } + + requeueRun( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.requeueRun(runId, args, tx); + } + + recordBulkActionMembership( + runId: string, + bulkActionId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.recordBulkActionMembership(runId, bulkActionId, tx); + } + + cancelRun( + runId: string, + data: { + completedAt?: Date; + error: TaskRunError; + bulkActionId?: string; + usageDurationMs?: number; + costInCents?: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.cancelRun(runId, data, args, tx); + } + + failRunPermanently( + runId: string, + data: { + status: TaskRunStatus; + completedAt: Date; + error: TaskRunError; + usageDurationMs: number; + costInCents: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.failRunPermanently(runId, data, args, tx); + } + + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + tx?: PrismaClientOrTransaction + ): Promise; + finalizeRun(...args: unknown[]): unknown { + return (this.delegate.finalizeRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + expireRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + snapshot: ExpireSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.expireRun(runId, data, args, tx); + } + + expireRunsBatch( + runIds: string[], + data: { error: TaskRunError; now: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.expireRunsBatch(runIds, data, tx); + } + + lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + parkPendingVersion( + runId: string, + data: { statusReason: string }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.parkPendingVersion(runId, data, args, tx); + } + + promotePendingVersionRuns( + runId: string, + args?: PromotePendingVersionArgs, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.promotePendingVersionRuns(runId, args, tx); + } + + expireParkedRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.expireParkedRun(runId, data, tx); + } + + suspendForCheckpoint( + runId: string, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.suspendForCheckpoint(runId, args, tx); + } + + resumeFromCheckpoint( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.resumeFromCheckpoint(runId, args, tx); + } + + rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rescheduleRun(runId, data, tx); + } + + enqueueDelayedRun( + runId: string, + data: { queuedAt: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.enqueueDelayedRun(runId, data, tx); + } + + rewriteDebouncedRun( + runId: string, + data: RewriteDebouncedRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rewriteDebouncedRun(runId, data, tx); + } + + updateMetadata( + runId: string, + data: { + metadata: string | null; + metadataType?: string; + metadataVersion: { increment: number }; + updatedAt: Date; + }, + options: { expectedMetadataVersion?: number }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.updateMetadata(runId, data, options, tx); + } + + clearIdempotencyKey( + params: ClearIdempotencyKeyInput, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.clearIdempotencyKey(params, tx); + } + + pushTags( + runId: string, + tags: string[], + where: { runtimeEnvironmentId: string }, + tx?: PrismaClientOrTransaction + ): Promise<{ updatedAt: Date }> { + return this.delegate.pushTags(runId, tags, where, tx); + } + + pushRealtimeStream( + runId: string, + streamId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.pushRealtimeStream(runId, streamId, tx); + } + + get primaryReadClient(): ReadClient { + return this.delegate.primaryReadClient; + } + + findRun( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise | null>; + findRun( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise | null>; + findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRun(...args: unknown[]): unknown { + return (this.delegate.findRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise>; + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise>; + findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRunOrThrow(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrow as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise | null>; + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise | null>; + findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise>; + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise>; + findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOrThrowOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrowOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + select: S; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + include: I; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise; + findRuns(...args: unknown[]): unknown { + return (this.delegate.findRuns as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIds( + ids: string[], + args: { select: S }, + client?: ReadClient + ): Promise>>; + findRunsByIds( + ids: string[], + args: { include: I }, + client?: ReadClient + ): Promise>>; + findRunsByIds(ids: string[], client?: ReadClient): Promise>; + findRunsByIds(...args: unknown[]): unknown { + return (this.delegate.findRunsByIds as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIdempotencyKeys( + args: { runtimeEnvironmentId: string; taskIdentifier: string; idempotencyKeys: string[] }, + client?: ReadClient + ): Promise { + return this.delegate.findRunsByIdempotencyKeys(args, client); + } + + createBatchTaskRunItem( + data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRunItem(data, tx); + } + + findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string + ): Promise | null> { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findExecutionSnapshot(args, client); + } + + findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createExecutionSnapshot(input, tx); + } + + findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointConnectedRunIds(waitpointId, client); + } + + findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointCompletedSnapshotIds(waitpointId, client); + } + + blockRunWithWaitpointEdges(params: { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.delegate.blockRunWithWaitpointEdges(params); + } + + countPendingWaitpoints( + waitpointIds: string[], + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.countPendingWaitpoints(waitpointIds, client, runId); + } + + countPendingWaitpointsWithPresence( + waitpointIds: string[], + client?: ReadClient + ): Promise<{ pendingIds: string[]; presentIds: string[] }> { + return this.delegate.countPendingWaitpointsWithPresence(waitpointIds, client); + } + + createWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.createWaitpoint(args, tx, opts); + } + + upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.upsertWaitpoint(args, tx, opts); + } + + findWaitpoint( + args: Prisma.SelectSubset, + client?: ReadClient, + opts?: WaitpointColocationOptions + ): Promise | null> { + return this.delegate.findWaitpoint(args, client, opts); + } + + findWaitpointOnPrimary( + args: Prisma.SelectSubset + ): Promise | null> { + return this.delegate.findWaitpointOnPrimary(args); + } + + findManyWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient, + runId?: string + ): Promise[]> { + return this.delegate.findManyWaitpoints(args, client, runId); + } + + updateWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.updateWaitpoint(args, tx, opts); + } + + updateManyWaitpoints( + args: Prisma.WaitpointUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyWaitpoints(args, tx); + } + + forWaitpointCompletion( + waitpointId: string, + context: ForWaitpointCompletionContext + ): Promise { + return this.delegate.forWaitpointCompletion(waitpointId, context); + } + + findManyTaskRunWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyTaskRunWaitpoints(args, client); + } + + deleteManyTaskRunWaitpoints( + args: Prisma.TaskRunWaitpointDeleteManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.deleteManyTaskRunWaitpoints(args, tx); + } + + findTaskRunAttempt( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findTaskRunAttempt(args, client); + } + + createTaskRunCheckpoint( + args: Prisma.SelectSubset, + ownerRunId?: string, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createTaskRunCheckpoint(args, ownerRunId, tx); + } + + createBatchTaskRun( + data: CreateBatchTaskRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRun(data, tx); + } + + updateBatchTaskRun( + args: { + where: Prisma.BatchTaskRunWhereUniqueInput; + data: Prisma.BatchTaskRunUpdateInput; + select: S; + }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.updateBatchTaskRun(args, tx); + } + + findBatchTaskRunById( + id: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunById(id, args, client); + } + + findBatchTaskRunByFriendlyId( + friendlyId: string, + environmentId: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByFriendlyId(friendlyId, environmentId, args, client); + } + + findBatchTaskRunByIdempotencyKey( + environmentId: string, + idempotencyKey: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + client + ); + } + + updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRun(args, tx); + } + + countBatchTaskRunItems( + where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus }, + client?: ReadClient + ): Promise { + return this.delegate.countBatchTaskRunItems(where, client); + } + + updateManyBatchTaskRunItems( + args: Prisma.BatchTaskRunItemUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRunItems(args, tx); + } + + findManyBatchTaskRunItems( + where: { taskRunId?: string; batchTaskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyBatchTaskRunItems(where, args, client); + } + + findBatchTaskRunItem( + where: { batchTaskRunId: string; taskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunItem(where, args, client); + } + + upsertWaitpointTag( + data: { environmentId: string; name: string; projectId: string; id?: string }, + tx?: PrismaClientOrTransaction, + // A tag has no owning run to co-locate with; when no minted `id` pins it by id-shape, a + // minted-new env's tags read this residency (NEW) so they land with the env's tokens/runs + // instead of defaulting to LEGACY. Single-store impls ignore it. + residency?: Residency + ): Promise { + return this.delegate.upsertWaitpointTag(data, tx, residency); + } + + findManyWaitpointTags( + args: { + where: Prisma.WaitpointTagWhereInput; + orderBy?: + | Prisma.WaitpointTagOrderByWithRelationInput + | Prisma.WaitpointTagOrderByWithRelationInput[]; + take?: number; + skip?: number; + }, + client?: ReadClient + ): Promise { + return this.delegate.findManyWaitpointTags(args, client); + } +} + +// `implements` above rejects a member of the interface that is missing here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not trip +// this. +type _ClassDeclaresNoExtraMembers = [Exclude] extends [ + never, +] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index a4109ab0104..07be12da94e 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -5,3 +5,8 @@ export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; export * from "./routingStoreMetrics.js"; export * from "./snapshotComparator.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..f1394be597e --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,193 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2964959c4fe..2339ab0dd5e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -1,7 +1,7 @@ import { createRedisClient, type Callback, - type Redis, + type RedisClient, type RedisOptions, type Result, } from "@internal/redis"; @@ -29,6 +29,19 @@ export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): strin .map((w) => w.id); } +/** + * The COMPLETE distinct set of completed-waitpoint ids, including those with no batch index. + * + * This is deliberately not `deriveOrder` deduped. `order` is the index oracle and carries only + * batch-indexed ids, because its positions ARE the indexes. A wait with no batch index (every + * `wait.for`, every single `triggerAndWait`, every token) has no position and is absent from it, + * while Postgres records it in the completed-waitpoint join like any other. Reading the id set back + * from `order` therefore loses exactly those waits, and a run resumed from Redis loses their results. + */ +export function deriveDistinctIds(completedWaitpoints: CompletedWaitpointRef[]): string[] { + return [...new Set(completedWaitpoints.map((w) => w.id))]; +} + // isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document. export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; @@ -162,6 +175,12 @@ export type SnapshotRead = { raw: string; cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; + /** + * The entry points at a cycle key that no longer exists, so its waitpoints are unreachable rather + * than absent. A caller must not treat this as an empty set: it has to fall back to Postgres, + * which still holds the join rows. + */ + danglingCycle?: boolean; }; export type AppendResult = @@ -186,8 +205,20 @@ export type SnapshotStoreMetrics = { recordLatency(op: string, ms: number): void; }; -export type RedisSnapshotStoreOptions = { - redisOptions: RedisOptions; +/** + * How the store reaches Redis. Exactly one of the two, enforced by the type rather than a runtime + * check: `never` on the opposite member makes both "neither" and "both" a compile error. + * + * `client` exists because production points at a Valkey/Redis CLUSTER, and cluster topology is not + * this package's business. Every command the store issues is key-addressed and every key carries a + * `{runId}` hashtag, so one slot serves a whole run and both endpoint shapes behave identically. + * A caller-supplied client is owned by the caller: `quit()` leaves it open. + */ +export type RedisSnapshotStoreConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; @@ -195,13 +226,25 @@ export type RedisSnapshotStoreOptions = { logger?: Logger; }; +/** + * Both window scripts return four leading slots before the first row: the id-cursor variant's + * `sinceRaw`, the head's order, the head's distinct set, and the head's dangling flag. Rows follow + * in four-element groups, so the head row is the group at this offset. + * + * Named because the offset drifted out of the comments describing it twice, and the second drift + * arrived in the change that fixed the first. + */ +const WINDOW_HEAD_ROW_INDEX = 4; + const SKIPPED = "skipped"; const FORKED = "forked"; const WRITTEN = "written"; const DUPLICATE = "duplicate"; export class RedisSnapshotStore { - private readonly redis: Redis; + private readonly redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly completedTtlMs: number; private readonly sinceLimit: number; @@ -215,15 +258,22 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), - }); + this.ownsClient = options.client === undefined; + this.redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), + }); this.#registerCommands(); } async quit(): Promise { // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit() // (or one after a failed connect) must never mask the real assertion failure. + // + // An injected client is the caller's. Closing it here would take down a connection shared with + // the sweeper or with another component, so a borrowed client is left open. + if (!this.ownsClient) return; if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -253,7 +303,19 @@ export class RedisSnapshotStore { completedWaitpoints: CompletedWaitpointRef[]; records?: CompletedWaitpointRecord[]; } - | { kind: "carryForward"; cycleSeq: number }; + | { + kind: "carryForward"; + cycleSeq: number; + /** + * The same refs a `new` cycle would carry. A carry the store refuses falls back to + * minting inside the same call, and it cannot do that without them: with no refs there is + * nothing to mint from, so the entry is written with no pointer, as before. + * + * Every production caller supplies them. Omitting them gives up the fallback. + */ + completedWaitpoints?: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + }; }): Promise { if (args.entry.completedWaitpoints !== undefined) { throw new Error( @@ -270,17 +332,29 @@ export class RedisSnapshotStore { let cycleMode = "none"; let cycleSeqIn = "0"; let orderJson = ""; + let distinctJson = ""; let records = ""; let orderCount = "0"; if (args.cycle?.kind === "new") { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; cycleSeqIn = String(args.cycle.cycleSeq); + + // Carried for the refusal path only. The script uses these solely when it declines the + // pointer and mints a replacement, and can only do that when the caller supplied them. + if (args.cycle.completedWaitpoints) { + const order = deriveOrder(args.cycle.completedWaitpoints); + orderJson = JSON.stringify(order); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; + orderCount = String(order.length); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + } } const reply = (await this.redis.appendSnapshotEntry( @@ -300,7 +374,8 @@ export class RedisSnapshotStore { records, orderCount, args.expectedCur ?? "", - args.expectedCur !== undefined ? "1" : "0" + args.expectedCur !== undefined ? "1" : "0", + distinctJson )) as string[]; return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); @@ -407,7 +482,17 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); - return decodeWaitpointIds(reply[0] === "1", reply[1] ?? ""); + // A dangling pointer means this entry's waitpoints are unreachable, not absent. Reporting + // `present: false` is what sends the caller to Postgres, which still holds the join rows. + if (reply[3] === "1") { + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore snapshot points at a cycle key that is gone", { + runId, + snapshotId, + }); + return { present: false, distinctIds: [], order: [] }; + } + return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -441,11 +526,13 @@ export class RedisSnapshotStore { } const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -455,15 +542,97 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -494,7 +663,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson, dangling] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -507,8 +676,16 @@ export class RedisSnapshotStore { if (pointer) { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; + if (dangling === "1") { + read.danglingCycle = true; + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore entry points at a cycle key that is gone", { + runId, + snapshotId: id, + }); + } if (orderKnown) { - const ids = decodeWaitpointIds(true, orderJson); + const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; this.#checkCycleMismatch(runId, Number(count), ids.order.length); } @@ -530,6 +707,24 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- A pointer whose cycle key is GONE. Not the same as having no pointer: this entry should + -- have waitpoints and cannot produce them, so a read must refuse rather than answer empty. + -- Reachable by eviction, and by the completion TTL, which is applied to every key for a run + -- at the same moment but lets them expire independently. + local function danglingFor(pointer) + if not pointer then return '0' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '0' end + if redis.call('EXISTS', wpKey(cs)) == 0 then return '1' end + return '0' + end + -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. + local function distinctFor(pointer) + if not pointer then return '' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '' end + return redis.call('HGET', wpKey(cs), 'distinct') or '' + end `; this.redis.defineCommand("appendSnapshotEntry", { @@ -549,6 +744,9 @@ export class RedisSnapshotStore { local orderCount = ARGV[11] local expectedCur = ARGV[12] local casEnabled = ARGV[13] == '1' + -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch + -- index, and those ids still have to come back on a read. + local distinctJson = ARGV[14] -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a @@ -579,27 +777,43 @@ export class RedisSnapshotStore { local cycleSeq = 0 local mismatch = 0 - if cycleMode == 'new' then - -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal - -- PEXPIRE loop from 1..c is correct. - cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) + + -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal + -- PEXPIRE loop from 1..c is correct. + local function mintCycle() + local minted = redis.call('HINCRBY', seqKey, 'c', 1) + redis.call('HSET', wpKey(minted), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then - redis.call('HSET', wpKey(cycleSeq), 'records', records) + redis.call('HSET', wpKey(minted), 'records', records) else -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key -- still holds another cycle's records, and order/count stay mutually consistent so the -- mismatch check cannot see it. No-op on a fresh key. - redis.call('HDEL', wpKey(cycleSeq), 'records') + redis.call('HDEL', wpKey(minted), 'records') end + return minted + end + + if cycleMode == 'new' then + cycleSeq = mintCycle() elseif cycleMode == 'carry' then - -- Attach a pointer only if this incarnation actually minted the cycle. seq can be - -- evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- Attach the CARRIED pointer only if this incarnation actually minted that cycle. seq can + -- be evicted while a wp: key survives, so a bare key-exists check would adopt a dead -- incarnation's order and records under a consistent count, invisibly. local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') if not c or minted < cycleSeqIn then + -- Refusing the pointer is right. Writing the entry WITHOUT one is not: it becomes the + -- head with no waitpoints, and a read of it answers present-with-nothing, which is the + -- one answer that tells the engine's repair it need not look. Mint a fresh cycle from + -- the refs the caller carried, in this same atomic call, so the entry always has a + -- pointer that can be trusted. The mismatch is still reported, for the metric. mismatch = 1 + -- Only possible when the caller carried the refs. With none there is nothing to mint + -- from, and the entry is written with no pointer, which is the older behaviour. + if distinctJson ~= '' then + cycleSeq = mintCycle() + end else cycleSeq = cycleSeqIn orderCount = c @@ -653,7 +867,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -665,7 +879,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -678,7 +892,78 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer), danglingFor(pointer) } + `, + }); + + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + -- + -- Both anchors, for the reason the append script gives: keys expire independently, and an + -- index lost to eviction while the entry hash survives would otherwise report an empty HIT + -- on every poll for the rest of the run's life, with Postgres holding the transitions. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '', '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) + end + return out `, }); @@ -708,7 +993,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '' } + local out = { sinceRaw, '', '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -722,7 +1007,14 @@ export class RedisSnapshotStore { end end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, @@ -730,9 +1022,26 @@ export class RedisSnapshotStore { } } -export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds { +export function decodeWaitpointIds( + present: boolean, + orderJson: string, + distinctJson = "" +): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - return { present, distinctIds: [...new Set(order)], order }; + + // The complete set is stored separately, because `order` omits every id with no batch index, so + // deduping the order to recover it silently drops every wait that has none. + // + // A cycle key always holds both fields, written by one command, so a missing `distinct` beside a + // NON-EMPTY `order` means the invariant is broken. Reconstructing from the order there would be + // the same lossy shortcut this field exists to remove, and the loss would be silent. Report the + // entry as not present instead, which sends the caller to Postgres. + if (distinctJson === "" && order.length > 0) { + return { present: false, distinctIds: [], order: [] }; + } + + const distinctIds: string[] = distinctJson === "" ? [] : (JSON.parse(distinctJson) as string[]); + return { present, distinctIds, order }; } declare module "@internal/redis" { @@ -755,6 +1064,7 @@ declare module "@internal/redis" { orderCount: string, expectedCur: string, casEnabled: string, + distinctJson: string, callback?: Callback ): Result; readSnapshotById( @@ -780,6 +1090,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..2b10bd766d0 --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,115 @@ +// The member names of RunStore, as data. +// +// The decorator suites enumerate this to drive one call per member. Member PRESENCE is not proved +// here: that is the compiler's job, through `implements RunStore` on the pass-through base and the +// assertions at the foot of this file. +// +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by `implements RunStore` on the generated class. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against `keyof RunStore`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [ + never, +] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..635160fd831 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,522 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. + * + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); +} + +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); +}); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..83abf43b179 --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,45 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts new file mode 100644 index 00000000000..90df7bfcc1d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -0,0 +1,160 @@ +// Production points at a Valkey/Redis CLUSTER. SCAN carries no key, so a cluster cannot route it: +// one connection iterates ONE node's keyspace and then returns a completed cursor. A single-client +// sweep would therefore report {scanned, expired, deleted, skipped} looking exactly like a clean +// pass, having examined roughly 1/N of the keyspace, and the rest would leak with nothing left to +// revisit it. Both sweep rules close unbounded leaks, and TRI-13453 gates the rollout dial on an +// OBSERVED sweep pass, so a false green here is the worst failure this component has. +// +// Everything the sweep does after the scan is key-addressed and a cluster client routes it without +// help, so the node list is the whole of the exposure. These tests pin that decision directly. +// +// There is no Redis-cluster container fixture in the repo (@internal/testcontainers ships slot +// arithmetic, not a cluster), so the cluster cases drive a real ioredis `Cluster` object that has +// never connected and assert which method the code reaches for. That is a test of our branch, not +// a simulation of Redis. A true multi-node integration test wants a cluster fixture, and the ticket +// building the cluster client is the one placed to add it. +import { describe, expect, it } from "vitest"; +import { Cluster, Redis } from "@internal/redis"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { clientPrefixOf, scanTargetsOf, SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; + +/** An ioredis Cluster that is never connected. `lazyConnect` keeps the constructor from dialling. */ +function offlineCluster(options?: { keyPrefix?: string }): Cluster { + return new Cluster([{ host: "127.0.0.1", port: 7000 }], { + lazyConnect: true, + redisOptions: options?.keyPrefix ? { keyPrefix: options.keyPrefix } : undefined, + }); +} + +describe("scanTargetsOf", () => { + it("returns the one connection for a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + try { + expect(scanTargetsOf(client)).toEqual([client]); + } finally { + client.disconnect(); + } + }); + + it("returns every master for a cluster, and never a replica", async () => { + const cluster = offlineCluster(); + const masters = [ + new Redis({ lazyConnect: true, port: 65001 }), + new Redis({ lazyConnect: true, port: 65002 }), + new Redis({ lazyConnect: true, port: 65003 }), + ]; + const replica = new Redis({ lazyConnect: true, port: 65004 }); + + const asked: string[] = []; + // The assertion that matters: the sweep asks for "master" specifically. Asking for "all" would + // scan replicas too and act twice on one keyspace. + (cluster as unknown as { nodes: (role: string) => Redis[] }).nodes = (role: string) => { + asked.push(role); + return role === "master" ? masters : [...masters, replica]; + }; + + expect(scanTargetsOf(cluster)).toEqual(masters); + expect(asked).toEqual(["master"]); + expect(scanTargetsOf(cluster)).not.toContain(replica); + + for (const client of [...masters, replica]) client.disconnect(); + cluster.disconnect(); + }); + + it("resolves the node list per call, so a failover is picked up", () => { + const cluster = offlineCluster(); + let generation = 0; + (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { + generation += 1; + return Array.from( + { length: generation }, + (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i }) + ); + }; + + expect(scanTargetsOf(cluster)).toHaveLength(1); + expect(scanTargetsOf(cluster)).toHaveLength(2); + cluster.disconnect(); + }); +}); + +describe("clientPrefixOf", () => { + it("reads the top-level keyPrefix on a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000, keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(client)).toBe("engine:"); + } finally { + client.disconnect(); + } + }); + + it("reads the nested redisOptions.keyPrefix on a cluster", () => { + // On a Cluster the prefix lives under redisOptions. Reading the top level yields "", every + // SCAN MATCH then misses, and the pass reports a clean sweep of nothing. + const cluster = offlineCluster({ keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(cluster)).toBe("engine:"); + } finally { + cluster.disconnect(); + } + }); + + it("is empty when no prefix is configured, for either shape", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + const cluster = offlineCluster(); + try { + expect(clientPrefixOf(client)).toBe(""); + expect(clientPrefixOf(cluster)).toBe(""); + } finally { + client.disconnect(); + cluster.disconnect(); + } + }); +}); + +describe("SweepResult.nodes", () => { + containerTest( + "a standalone pass reports the one connection it covered", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + const result = await sweeper.sweep({ dryRun: true }); + // Without this field a pass that covered one node of six is indistinguishable from a + // complete one, which is exactly the false green the fan-out exists to prevent. + expect(result.nodes).toBe(1); + } finally { + await sweeper.quit(); + } + } + ); +}); + +describe("client ownership", () => { + containerTest("quit() leaves a borrowed client open", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const client = new Redis(redisOptions); + const sweeper = new SnapshotOrphanSweeper({ + client, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + await sweeper.quit(); + // The store and the sweep can share one cluster client. If quit() closed a client it did not + // open, the first component to shut down would take the other one's connection with it. + await client.set(`ownership:${generateInternalId()}`, "1"); + expect(await client.ping()).toBe("PONG"); + } finally { + client.disconnect(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts new file mode 100644 index 00000000000..40c90bc4774 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -0,0 +1,360 @@ +// Rule 2 deletes a whole keyspace on the strength of "findRunsByIds returned no row for it". The +// catch in #sweepBatch covers a lookup that THROWS; it cannot see a lookup that succeeds and is +// incomplete, and a row that exists but did not come back reads exactly like a run that never +// existed. `findRunsByIds` partitions ids by residency and asks each store only for its own, and +// with no client passed it reads each store's replica — both sound today, but neither is something +// this delete path can verify. +// +// A false negative leaks keys, which is bounded and recoverable. A false positive destroys a live +// run's execution state. So deletion requires two sightings across the confirm window, and these +// tests pin that: a single pass never deletes, however old the keyspace. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("rule 2 requires a second sighting", () => { + containerTest( + "one pass marks an orphan and deletes nothing, however old the keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A thousand times the age gate. Age is not what holds the deletion back. + const ancient = new Date(Date.now() - 1000 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + + const first = await sweeper.sweep(); + + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const second = await sweeper.sweep(); + + expect(second.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a marked keyspace is not deleted until the confirm window has passed", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // An hour. Passes minutes apart must not convert a candidate. + confirmOrphanAfterMs: 60 * 60 * 1000, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + await sweeper.sweep(); + const second = await sweeper.sweep(); + const third = await sweeper.sweep(); + + expect(second.deleted).toBe(0); + expect(second.pendingDeletion).toBe(1); + expect(third.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a transient miss followed by a found run does not delete, and does not leave the keyspace pre-authorised", + async ({ prisma, redisOptions }) => { + // The case the guard exists for. Pass 1 gets an incomplete answer and marks the keyspace. + // Pass 2 sees the run, so it must clear the mark: were the mark to survive, a LATER genuine + // absence would delete on its own first sighting and the two-sighting rule would be gone. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? // Succeeds and is incomplete: exactly what the catch cannot see. + Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // The run is alive and terminal in Postgres the whole time. Only the lookup lies. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const first = await sweeper.sweep(); + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + + lie = false; + const second = await sweeper.sweep(); + expect(second.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + // The run vanishes for real. With the mark cleared this is a first sighting again. + lie = true; + const third = await sweeper.sweep(); + expect(third.deleted).toBe(0); + expect(third.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const fourth = await sweeper.sweep(); + expect(fourth.deleted).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("the marker cannot expire out from under a candidate", () => { + containerTest( + "a marked keyspace still carries its mark after the whole keyspace is re-read", + async ({ prisma, redisOptions }) => { + // The marker used to be a key with its own TTL derived from the confirm window, which could + // be shorter than the interval between passes: the marker written at T was gone by + // T+interval, every pass wrote a fresh one, and rule 2 deleted nothing while reporting clean. + // It is now a field on the run's `seq` hash, so it lives exactly as long as the keyspace and + // there is no lifetime left to misconfigure. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + // The mark is a field on seq, and it carries no expiry of its own. + expect(await probe.hget(snapshotKeys(runId).seq, "orph")).not.toBeNull(); + expect(await probe.pttl(snapshotKeys(runId).seq)).toBe(-1); + + expect((await sweeper.sweep()).deleted).toBe(1); + // And it went with the keyspace rather than outliving it. + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a run seen alive clears its marker", () => { + containerTest( + "a lie, then a LIVE run, then a lie again does not delete", + async ({ prisma, redisOptions }) => { + // The hole a long marker lifetime opens. Only terminal runs used to clear the marker, so a + // keyspace marked by an incomplete lookup and then seen ALIVE kept its mark. A later genuine + // absence would then find a mature marker and delete on what is really a first sighting. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // EXECUTING, not terminal. This run never reaches rule 1, so rule 1 cannot be what clears + // the mark; a SUSPENDED run can legitimately sit here for weeks. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "EXECUTING" }, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + lie = false; + const seenAlive = await sweeper.sweep(); + expect(seenAlive.skipped).toBeGreaterThan(0); + expect(seenAlive.deleted).toBe(0); + + lie = true; + const afterAlive = await sweeper.sweep(); + // A first sighting again, because being seen alive cleared the mark. + expect(afterAlive.deleted).toBe(0); + expect(afterAlive.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a pass can stop inside a budget", () => { + containerTest( + "an already-passed deadline yields a partial pass", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } + } + ); + + containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + const controller = new AbortController(); + controller.abort(); + + try { + const result = await sweeper.sweep({ signal: controller.signal }); + expect(result.partial).toBe(true); + } finally { + await sweeper.quit(); + } + }); + + containerTest("a pass with budget to spare is not partial", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() + 60_000 }); + expect(result.partial).toBe(false); + } finally { + await sweeper.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..7d7eb2aca0d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,469 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + // A dry pass writes no marker, so an unconfirmed rule 2 candidate reports as pending rather + // than as a deletion. That is what a real pass would do at this instant, which is the honest + // answer for a preview: nothing is confirmed yet, so nothing would be deleted yet. + expect(result.deleted).toBe(0); + expect(result.pendingDeletion).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "still finds keyspaces when the client carries a key prefix", + async ({ prisma, redisOptions }) => { + // ioredis prepends its keyPrefix to keys for ordinary commands, but NOT to a SCAN MATCH + // pattern, and it returns matched keys with the prefix still on them. The engine sets a + // prefix on every other Redis client it builds, so a sweep that ignored this would match + // nothing and report a clean pass: a safety net that silently protects nothing. + const prefixed = { ...(redisOptions as object), keyPrefix: "engine:" } as never; + const store = new RedisSnapshotStore({ + redisOptions: prefixed, + completedTtlMs: COMPLETED_TTL_MS, + }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: prefixed, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(prefixed, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.scanned).toBe(1); + expect(result.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + expect(await probe.exists(`snap:{${runId}}:wp:1`)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + } + + await sweeper.sweep({ batchSize: 2 }); + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..86bb084d5c9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,567 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { + Cluster, + createRedisClient, + type Redis, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; + +/** + * The keyspace prefix, owned by `snapshotKeys` in the store rather than configurable here. A sweep + * that could be pointed at a different prefix would be a fiction: the store writes `snap:` keys + * unconditionally, so there is no other keyspace to point it at. + */ +const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; +const DEFAULT_BATCH_SIZE = 1000; + +/** + * How long a rule 2 candidate must have been marked before it may be deleted. It has to exceed the + * interval between passes, or a candidate is never sighted twice and never converts. + */ +const DEFAULT_ORPHAN_CONFIRM_MS = 60 * 60 * 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; + /** + * Rule 2 candidates that were marked but not deleted, because deletion needs a second sighting + * in a later pass. A number that never converts to `deleted` means the confirm window is longer + * than the interval between passes, or the marker TTL is shorter than it. + */ + pendingDeletion: number; + /** + * Connections the pass iterated: every master of a cluster, or 1 standalone. Reported because the + * failure this component cannot tolerate is a false green, and a pass that covered one node of + * six is indistinguishable from a complete one by any other field here. TRI-13453 gates the dial + * on an observed sweep pass, so the observation has to carry its own coverage. + */ + nodes: number; + /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ + partial: boolean; +}; + +/** + * Exactly one of `redisOptions` or `client`, enforced by the type rather than a runtime check. + * With `redisOptions` the sweep opens its OWN connection, which is the preferred shape: a long + * scan can then never stall a hot-path client. `client` exists for a caller that has already built + * a client and wants the sweep to use it; a borrowed client is left open by `quit()`. + * + * What the sweep needs is a connection of its OWN, not one it built itself. A caller pointing at a + * cluster should build a SECOND, sweep-dedicated cluster client and pass it here: that keeps a long + * scan off the hot path just as well as `redisOptions` does. Handing over the client the snapshot + * store is using is the case to avoid. + */ +export type SnapshotOrphanSweeperConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type SnapshotOrphanSweeperOptions = SnapshotOrphanSweeperConnection & { + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + /** + * How long a rule 2 candidate must stay marked before the sweep will delete it. Defaults to one + * hour. + * + * Set it at or below the interval between passes, or the second sighting arrives too early to + * count and every candidate needs three passes instead of two. It does NOT need to exceed the + * interval; the constraint people reach for ("longer than the interval") is the wrong one and + * only costs latency. + * + * There is no marker-lifetime constraint to satisfy alongside it. The marker is a field on the + * run's `seq` hash, so it lives exactly as long as the keyspace it describes: it cannot expire + * out from under a candidate that is still waiting for its second sighting, and it cannot outlive + * a keyspace that was deleted. + */ + confirmOrphanAfterMs?: number; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + readonly #ownsClient: boolean; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #confirmOrphanAfterMs: number; + /** + * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys + * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return + * matched keys with it still attached. Unhandled, a prefixed client makes the sweep match nothing + * and report a clean pass, which is the worst outcome for a safety net. + */ + readonly #clientPrefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#confirmOrphanAfterMs = options.confirmOrphanAfterMs ?? DEFAULT_ORPHAN_CONFIRM_MS; + this.#ownsClient = options.client === undefined; + this.#redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => + this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + this.#clientPrefix = clientPrefixOf(this.#redis); + } + + async quit(): Promise { + // A borrowed client belongs to the caller; closing it here would take down a connection the + // snapshot store may still be using. + if (!this.#ownsClient) return; + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + * + * `deadline` and `signal` let the caller stop a pass cleanly instead of having it killed + * mid-cursor. The scheduler needs this: redis-worker moves a dequeued item's score to + * `now + visibilityTimeoutMs` and nothing extends it, so a pass that outlives its timeout is + * redelivered and runs concurrently with itself. A pass that stops inside its budget cannot. + * + * Whichever way it stops, `partial` comes back true. Reporting a truncated pass as a full one is + * the same false green as under-scanning a cluster: TRI-13453 gates the dial on an OBSERVED + * sweep pass, so the observation has to say how much of the keyspace it actually reached. + */ + async sweep(opts?: { + batchSize?: number; + dryRun?: boolean; + /** Epoch ms. The pass stops at the next batch boundary once passed. */ + deadline?: number; + signal?: AbortSignal; + }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { + scanned: 0, + expired: 0, + deleted: 0, + skipped: 0, + pendingDeletion: 0, + nodes: 0, + partial: false, + }; + + // Checked at batch boundaries only. Stopping mid-batch would leave a run half-acted-on, and a + // batch is bounded work, so the boundary is both the safe and the timely place. + const outOfBudget = () => + opts?.signal?.aborted === true || + (opts?.deadline !== undefined && Date.now() >= opts.deadline); + + // SCAN carries no key, so a cluster cannot route it: one connection iterates ONE node's + // keyspace and then reports a completed cursor. A single-client sweep against a cluster would + // therefore return a clean-looking result having examined roughly 1/N of the keyspace, and the + // rest would leak with nothing to revisit it. Both rules are unbounded leaks when missed, so + // the pass fans out over every master and only reports done when all of them are done. + const nodes = this.#scanTargets(); + result.nodes = nodes.length; + + for (const node of nodes) { + if (outOfBudget()) { + result.partial = true; + break; + } + + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await node.scan( + cursor, + "MATCH", + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + + if (outOfBudget()) { + // A cursor mid-iteration means this node is not finished, so the pass is not either. + result.partial = true; + break; + } + } while (cursor !== "0"); + + if (result.partial) break; + } + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + #scanTargets(): Redis[] { + return scanTargetsOf(this.#redis); + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + // Every run that EXISTS clears its rule 2 marker, live ones included. It has to be every one, + // not just the terminal ones: a keyspace marked by an incomplete lookup, then seen alive, then + // missed again would otherwise present a mature marker on what is really a first sighting, and + // the two-sighting rule would be gone exactly when it was needed. This costs one DEL per + // existing run per pass, which is the price of the guard being sound rather than nearly sound. + const present = runIds.filter((runId) => rows.has(runId)); + if (!dryRun && present.length > 0) { + // Individual commands, never one pipeline: these keys span runs, so they span cluster slots. + await Promise.all(present.map((runId) => this.#clearOrphanMarker(runId))); + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** + * Rule 2: a keyspace with no run row at all, past the age threshold. + * + * TWO SIGHTINGS ARE REQUIRED. The `catch` in #sweepBatch covers a lookup that THROWS, but it + * cannot see a lookup that succeeds and is incomplete: a row that exists but did not come back + * reads exactly like a run that never existed, and the response to that is deleting a live run's + * execution state. `findRunsByIds` routes through RoutingRunStore.#findRunsByIdSet, which + * partitions ids by residency and asks each store only for its own — and with no client passed it + * reads each store's REPLICA. Both are sound today (id classification is authoritative for runs, + * and replica lag is nowhere near the 24h age gate), but each is an assumption held somewhere + * else in the codebase, not something this delete path can check. + * + * The asymmetry decides it: a false negative leaks keys, which is bounded and recoverable, while + * a false positive destroys live state. So an absent row marks the keyspace and returns; only a + * candidate still absent in a LATER pass is deleted. Any transient incomplete answer, whatever + * its cause, has to occur twice across the confirm window to do damage. + */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + const seqKey = snapshotKeys(runId).seq; + const markedAtRaw = await this.#redis.hget(seqKey, ORPHAN_MARKER_FIELD); + const markedAt = markedAtRaw === null ? undefined : Number(markedAtRaw); + + if (markedAt === undefined || Number.isNaN(markedAt)) { + if (!dryRun) { + // The field carries no TTL of its own; it lives and dies with the seq hash, which the + // keyspace's own completion expiry already governs. That removes the marker-lifetime knob + // whose derivation was wrong in the first place. + await this.#redis.hset(seqKey, ORPHAN_MARKER_FIELD, String(Date.now())); + } + result.pendingDeletion += 1; + return; + } + + if (Date.now() - markedAt < this.#confirmOrphanAfterMs) { + result.pendingDeletion += 1; + return; + } + + if (!dryRun) { + // One slot: every key here carries the same `{runId}` hash tag. The marker is a field on + // `seq`, which is in `keys`, so it goes with the keyspace rather than needing its own entry. + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** + * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later + * genuine absence still needs its own two sightings rather than inheriting a stale one. + * + * Called for EVERY run row the lookup returned, live ones included, and it has to be: a keyspace + * marked by an earlier incomplete lookup can belong to a run that is perfectly alive, and a + * SUSPENDED run can sit that way for weeks. Leaving the marker in place would let a later genuine + * absence delete on what is really a first sighting, which is the hole the two-sighting rule + * exists to close. It costs one DEL per existing run per pass; that is the price of the guard + * being sound rather than nearly sound. + */ + async #clearOrphanMarker(runId: string): Promise { + try { + await this.#redis.hdel(snapshotKeys(runId).seq, ORPHAN_MARKER_FIELD); + } catch { + // Best effort. A marker that outlives its usefulness expires on its own TTL. + } + } + + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${SNAPSHOT_KEYSPACE_PREFIX}{${runId}}:wp:${n}`); + } + + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + // One round trip for the whole set, rather than one per candidate. Cluster-safe: every key here + // carries the same `{runId}` hash tag, so the whole pipeline lands in one slot on one node. + // The same holds for the pexpire pipeline and the multi-key DEL above. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + + /** + * The run id is whatever sits inside the hash tag, so a client prefix on the returned key does not + * need stripping: `engine:snap:{run_x}:e` and `snap:{run_x}:e` both yield `run_x`. + */ + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +/** + * Every connection a pass must iterate to cover the whole keyspace: each master of a cluster, or + * the one standalone connection. Replicas are excluded — they hold the same keys as their master, + * so scanning them would double-count and act on one keyspace twice. + * + * Module-level and exported so the fan-out decision can be pinned on its own. It is the whole of + * the defect this guards against: everything the sweep does AFTER the scan is key-addressed and a + * cluster client routes it correctly without help, so the node list is the only place a cluster + * can silently cost the pass coverage. + * + * Resolved per pass, never cached: cluster topology changes under failover and resharding, and a + * stale node list is the same silent under-scan this exists to prevent. + */ + +/** + * Rule 2's "seen absent once" marker is a FIELD on the run's `seq` hash, not a key of its own. + * + * As a separate key its removal depended on the deleting call site remembering to append it to the + * DEL, which is the kind of contract a later edit breaks with no test noticing: a marker outliving + * its keyspace would let a recreated keyspace be deleted on what is really a first sighting. `seq` + * is already in `#allKeys`, so as a field the marker cannot outlive the keyspace at all. + */ +const ORPHAN_MARKER_FIELD = "orph"; + +export function scanTargetsOf(client: RedisClient): Redis[] { + return client instanceof Cluster ? client.nodes("master") : [client]; +} + +/** + * The ioredis client-level prefix for either endpoint shape. On a Cluster it lives on the nested + * `redisOptions`, not on the top-level options, and reading the wrong one yields "" — which makes + * every SCAN MATCH miss and the pass report a clean sweep of nothing. + */ +export function clientPrefixOf(client: RedisClient): string { + if (client instanceof Cluster) { + return (client.options.redisOptions?.keyPrefix as string | undefined) ?? ""; + } + return (client.options.keyPrefix as string | undefined) ?? ""; +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..258731a60b5 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..12ab76a0f26 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..437872106e7 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,100 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +/** + * Records what the decorator forwarded, and answers with a per-member sentinel. No database is + * involved in whether mode off is a pass-through, so none is started; the behavioural suites for + * every other mode run against a real Postgres and a real Redis. + */ +function forwardingProbe(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + return (...args: unknown[]) => { + calls.push(prop); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = forwardingProbe(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = forwardingProbe(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = forwardingProbe().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = forwardingProbe(); + const modes = ["dual-write", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..58c80b4df72 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,104 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("ignores the dial at redis-only, whatever it is set to", () => { + // Postgres holds no snapshot rows at that position, so a run routed away from Redis reads + // nothing at all. The percentage is meaningful only while both stores hold the data. + for (const percent of [0, 1, 50, 99]) { + const store = probe("redis-only", percent); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..a458e9070ed --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,420 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + containerTest( + "returns the same field set Postgres does, key for key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + // Not a value comparison: a column the hydrator forgets is absent rather than wrong, so it + // shows up as a missing KEY. lastHeartbeatAt was omitted this way and read back undefined + // where Postgres returns null, on every Redis-served read. + expect(Object.keys(fromRedis!).sort()).toEqual(Object.keys(fromPostgres!).sort()); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts new file mode 100644 index 00000000000..b308cb20988 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts @@ -0,0 +1,321 @@ +// `redis-only` is the terminal cutover, and it is the only dial position where Postgres stops being +// authoritative: it cannot be rolled back by turning the dial down, because the snapshots written +// while it was on exist nowhere else. It is also the only position that is a PAIR of settings, not +// one — the decorator's mode AND `snapshotWrites: false` on the store underneath it — and the two +// are set by different tickets. Every test here builds the pair, because testing the mode against a +// store that still writes snapshots would exercise a configuration that never ships. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** The shipping pair: decorator at `redis-only` over a store that writes no snapshot rows. */ +function build(prisma: never, redisOptions: never) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +function birth(env: SnapshotFixtureEnv, id: string) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +async function seedRun(decorated: TaskRunExecutionSnapshotStore, env: SnapshotFixtureEnv) { + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birth(env, snapshotId), + }); + return { runId, snapshotId }; +} + +function transition(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("redis-only: Postgres stops holding snapshots", () => { + containerTest("the run row lands but no snapshot row does", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + // The run itself is still Postgres-authoritative at this position. Only its snapshots move. + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + + // And the snapshot is genuinely in Redis under the id the caller minted. + const head = await redis.getLatest(runId); + expect(head?.id).toBe(snapshotId); + } finally { + await redis.quit(); + } + }); + + containerTest("transitions write no snapshot row either", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + await decorated.createExecutionSnapshot(transition(runId, env, "Run continued")); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const since = await redis.getSinceCreatedAt(runId, new Date(Date.now() - 60_000), { + limit: 50, + }); + expect(since.kind).toBe("hit"); + expect(since.kind === "hit" ? since.entries.length : 0).toBeGreaterThanOrEqual(2); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a completion still updates the run row while writing no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.completeAttemptSuccess( + runId, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: generateInternalId(), + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + // The mutation half of a nested write must still land, or the run never finishes. + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(run.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "no completed-waitpoint join rows are written for a snapshot Postgres does not have", + async ({ prisma, redisOptions }) => { + // The join rows point at a snapshot row. With snapshot writes off there is no such row, so + // inserting them would leave links dangling at a snapshot only Redis holds. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ], + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const joins = await prisma.$queryRawUnsafe<{ n: bigint }[]>( + `SELECT count(*) AS n FROM "_completedWaitpoints" WHERE "B" = ANY($1::text[])`, + [wpA, wpB] + ); + expect(Number(joins[0]!.n)).toBe(0); + } finally { + await redis.quit(); + } + } + ); +}); + +describe("redis-only: every read is served from Redis", () => { + containerTest( + "the hot read, the since window and the waitpoint lookups all come from Redis", + async ({ prisma, redisOptions }) => { + // At every earlier position a Redis miss falls back to Postgres and the caller never notices. + // Here Postgres holds nothing, so a read that fell back would answer empty rather than wrong, + // and a run would silently lose its state. Each read is asserted to be Redis-sourced. + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const created = await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [{ id: wpA, index: 0 }], + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.id).toBe(created.id); + expect(latest!.completedWaitpointOrder).toEqual([wpA]); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(window.length).toBeGreaterThan(0); + + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + + expect(reads.length).toBeGreaterThan(0); + expect(reads.every((r) => r.source === "redis")).toBe(true); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an unrecognised read shape falls through to a Postgres that holds nothing", + async ({ prisma, redisOptions }) => { + // CHARACTERISATION, NOT AN ENDORSEMENT. `findManyExecutionSnapshots` serves from Redis only + // for the since-window shape `matchSinceWindow` recognises; anything else delegates. At every + // dial position before this one that is harmless, because Postgres holds the same rows. Here + // it holds none, so the caller gets an EMPTY result rather than an error, and empty is a + // valid answer to this query. The same is true of the `miss` and `danglingCycle` fallbacks in + // that method: all three are safe everywhere except the one position that cannot fall back. + // + // Only the engine's own call shapes reach this method today, and it issues the since-window + // one, so nothing is broken. It is pinned here so the terminal-cutover ticket decides + // deliberately whether a fall-through at `redis-only` should throw instead of answering + // empty, rather than discovering this shape in production. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + + // No `createdAt` cursor, so the shape does not match and the read is delegated. + const unmatched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(unmatched).toEqual([]); + + // The same run, asked the shape the engine actually issues, answers in full from Redis. + const matched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(matched.length).toBeGreaterThan(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the read cohort dial cannot route a run away from Redis", + async ({ prisma, redisOptions }) => { + // readPercent is a ramp control for `redis-read`. At `redis-only` a run routed to Postgres + // would read a database that holds no snapshots at all, so the dial must be ignored here + // whatever it is set to. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: string[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + readPercent: 0, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (_m, source) => reads.push(source), + }, + } + ); + + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest!.id).toBe(snapshotId); + expect(reads).not.toContain("postgres"); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..9952f569e2b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,286 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the fork guard on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + // A stale expectation: this names a head that was never current. Outside a transaction the + // append is rejected as forked and never written. Staging must not weaken that, or a write + // the store would have refused becomes the head purely because it ran inside a transaction. + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "stale"), previousSnapshotId: generateInternalId() }, + tx + ); + }); + + expect(await redis.getById(runId, id)).toBeNull(); + + const head = await redis.getLatest(runId); + expect(head?.id).not.toBe(id); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "honours a correct expectation on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const head = await redis.getLatest(runId); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "expected"), previousSnapshotId: head!.id }, + tx + ); + }); + + expect((await redis.getById(runId, id))?.entry.description).toBe("expected"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..e6468c4f09f --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,454 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..c6bc145a91d --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,980 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveDistinctIds, deriveOrder } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + ReadClient, + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * A `compare` position was named here before its behaviour existed, and it read from this type as a + * real dial position while behaving in every respect exactly like `dual-write`. A dial value that + * silently does something other than its name is worse than a missing one: turning it on would have + * looked like enabling divergence reporting and delivered plain dual-write. It is added back by the + * ticket that implements the sampled dual-read and diff, at which point the name will be true. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + /** + * The head this append expects, carried through staging so the compare-and-set survives the + * deferral. Dropping it would silently disable the fork guard for every snapshot written inside a + * transaction, and a stale append that should be rejected would instead become the head. + */ + expectedCur?: string; + completedWaitpoints?: CompletedWaitpointRef[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: StagedAppend[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: StagedAppend[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + item.expectedCur, + item.completedWaitpoints + ); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + // Carry the staging buffer through. Without it, a handle taken inside a transaction appends + // immediately, which is the exact ordering the facade exists to prevent. + return this.#wrap(store, this.staging); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; + + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // Built from the COMPLETE id set, which is what the delegate connects in Postgres, with the + // index taken from the ordered list where the id appears in it. Building from the ordered list + // instead would drop every id with no batch index, exactly the ids Postgres still records. + lockCycleRefs( + withStamp.snapshot.completedWaitpointIds, + withStamp.snapshot.completedWaitpointOrder + ) + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push({ + entry, + ...(expectedCur !== undefined && { expectedCur }), + ...(completedWaitpoints && { completedWaitpoints }), + }); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number; completedWaitpoints: CompletedWaitpointRef[] } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + const distinct = deriveDistinctIds(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previousIds = head?.completedWaitpointIds; + + // Both halves must match. Comparing the order alone is not enough: it holds only indexed ids, + // so two DIFFERENT single waits both present an empty order and would compare equal, and the + // second would inherit the first's waitpoint set instead of minting its own. + if ( + head?.cycle && + previousIds && + sameOrder(previousIds.order, order) && + sameSet(previousIds.distinctIds, distinct) + ) { + return { + kind: "carryForward", + cycleSeq: head.cycle.cycleSeq, + completedWaitpoints, + }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + + // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that + // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here + // makes that misconfiguration unreachable rather than merely documented. + if (this.mode === "redis-only") return true; + + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + if (read.danglingCycle) { + // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would + // hand back an empty set that looks authoritative, and the run would resume with no waits. + // Postgres still has the join rows. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + if (result.entries.some((entry) => entry.danglingCycle)) { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { hydrateWaitpointRows?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + // Three cases, and only the last needs a second Redis call. The read already carries the ids + // when the store decoded them. An entry with no wait cycle has no waitpoints by construction, + // which is the common case and used to cost a round trip to rediscover. Anything else asks. + const ids = + read.completedWaitpointIds ?? + (read.cycle === undefined + ? { present: true, distinctIds: [], order: [] } + : await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; + + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + // A column no code writes, so Postgres returns null for it on every row. The entry does not + // carry it, and omitting it here would hand back undefined where Postgres hands back null, + // on every single read served from Redis. + lastHeartbeatAt: null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} + +/** + * Turns the lock site's two lists into cycle refs. `completedWaitpointIds` is the complete set the + * delegate connects; `completedWaitpointOrder` gives a position only to the ids that have one, and a + * repeated id keeps each of its positions. + */ +function lockCycleRefs(ids: string[], order: string[]): { id: string; index?: number }[] { + const refs: { id: string; index?: number }[] = []; + const indexed = new Set(); + + order.forEach((id, index) => { + refs.push({ id, index }); + indexed.add(id); + }); + + for (const id of ids) { + if (!indexed.has(id)) refs.push({ id }); + } + + return refs; +} + +/** Membership only, for the id set, which has no meaningful order. */ +function sameSet(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + const seen = new Set(a); + return b.every((id) => seen.has(id)); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..4bc2d0dc271 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,706 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps a completed waitpoint that has no batch index", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // Every wait.for, every single triggerAndWait and every token resumes with no batch index: + // the engine passes `index: b.batchIndex ?? undefined`. Postgres records the id in the + // completed-waitpoint join regardless. The ordered list cannot hold it, because its + // positions ARE the indexes, so the complete set has to be stored separately or the wait's + // result vanishes on a Redis read. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "single wait") + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + // No position, so it is absent from the oracle. That part is correct. + expect(ids.order).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "matches the Postgres join for a mix of indexed and index-less waits", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB, wpC] = await seedSnapshotWaitpoints(prisma, env, 3); + + const created = await decorated.createExecutionSnapshot( + resumeInput( + runId, + env, + [{ id: wpA, index: 0 }, { id: wpB }, { id: wpC, index: 1 }], + "mixed wait" + ) + ); + + // Parity with what Postgres holds is the actual requirement: the engine iterates the rows + // this set fetches, and uses the order only to assign each one its index. + const fromRedis = await redis.getSnapshotWaitpointIds(runId, created.id); + const fromPostgres = await new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }).findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + + expect([...fromRedis.distinctIds].sort()).toEqual([...fromPostgres].sort()); + expect(fromRedis.order).toEqual([wpA, wpC]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "two consecutive index-less waits do not share a cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + // Neither wait has a batch index, so both present an EMPTY order. Deciding carry-forward on + // the order alone makes them compare equal, and the second silently inherits the first's + // waitpoint set: its own result is never stored and a read returns the wrong id. + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "first single wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB }], "second single wait") + ); + + expect((await redis.getSnapshotWaitpointIds(runId, first.id)).distinctIds).toEqual([wpA]); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).distinctIds).toEqual([wpB]); + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same index-less wait repeated does still carry forward", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // The copy-forward case must survive the stricter comparison: the same id set, still one key. + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "wait")); + const carried = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "carry") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + expect((await redis.getSnapshotWaitpointIds(runId, carried.id)).distinctIds).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the dequeue snapshot keeps an index-less waitpoint", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + // Postgres connects completedWaitpointIds, the COMPLETE set. Building the Redis refs from + // completedWaitpointOrder instead drops every id that has no position in it. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect([...ids.distinctIds].sort()).toEqual([wpA, wpB].sort()); + expect(ids.order).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot hydrates an index-less waitpoint row", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "single")); + + // The hot read hydrates the rows from the id set, so an incomplete set means the resume + // gets no waitpoint at all. + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a refused carry mints a fresh cycle rather than writing a pointerless head", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const snapshotId = generateInternalId(); + + // Driven at the store, because the decorator cannot reach this state deliberately: its + // probe reads the head first, sees the id set no longer matches, and mints a new cycle. + // The refusal is only reachable when the key vanishes BETWEEN that probe and the append, + // which is a race. Naming a cycle this incarnation never minted reproduces the same + // refusal deterministically. + const result = await redis.append({ + entry: { + id: snapshotId, + engine: "V2", + executionStatus: "EXECUTING", + description: "carry a cycle that was never minted", + runId, + runStatus: "EXECUTING", + createdAt: new Date().toISOString(), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + kind: "transition", + isTerminal: false, + cycle: { + kind: "carryForward", + cycleSeq: 9999, + completedWaitpoints: [{ id: wpA, index: 0 }], + }, + }); + + expect(result.outcome).toBe("written"); + if (result.outcome !== "written") return; + + // Refusing the pointer is right. Writing the entry with NO pointer is not: it becomes the + // head, and a read of it answers present-with-nothing, which is the one answer that stops + // the engine's read-repair from looking. + expect(result.cycleMismatch).toBe(true); + expect(result.cycleSeq).toBeGreaterThan(0); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a dangling pointer reads as not present, not as an empty set", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + + // The entry keeps its pointer and the cycle key goes. Reachable by eviction, and by the + // completion TTL, which is set on every key for a run at once but expires them separately. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + // present:false is what sends the caller to Postgres, which still holds the join rows. + // present:true with an empty set would suppress the engine's read-repair. + expect(ids.present).toBe(false); + expect(ids.distinctIds).toEqual([]); + + // And the projections the engine actually calls fall back rather than answering empty. + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the hot read falls back to Postgres when the cycle key is gone", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // Served from Postgres, so the waitpoint is still there and the resume is not silently + // stripped of it. + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window falls back to Postgres when the head cycle key is gone", + async ({ prisma, redisOptions }) => { + // The hot read has always handled this. The since-window did not: its Lua returned the head's + // order and distinct set but never its dangling flag, so the decorator's fallback guard was + // dead code and an expired cycle key came back as an EMPTY order. Empty means "no indexed + // waitpoints" to the engine, which is how a batched triggerAndWait resumes with every + // position lost rather than falling back to the store that still knows them. + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The entry hash survives; only the cycle key goes. The completion TTL is applied per key, + // so this is a state the keyspace reaches on its own. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..e0b8a1a2107 --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,167 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 3ceb0a462dd..9ea39473e5b 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -31,6 +31,11 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -45,6 +50,14 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -66,6 +79,14 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -77,6 +98,14 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -87,6 +116,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -294,6 +328,14 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus; From 4c163874260585f4711d98cc2ab6833281f73ab3 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski Date: Wed, 26 Aug 2026 15:26:44 +0200 Subject: [PATCH 23/28] =?UTF-8?q?fix(webapp):=20project=20integrations=20p?= =?UTF-8?q?age=20=E2=80=94=20Staging=20gating,=20unreachable=20code,=20and?= =?UTF-8?q?=20follow-ups=20(#4784)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs on the project integrations page, one commit each for the two reported ones and four for the follow-ups found while fixing them. ## `chore`: remove unreachable code on the integrations page (TRI-12645) Two notification panels in `VercelSettingsPanel` could never render: 1. The **"Failed to load Vercel settings"** panel was gated on a `hasError` state whose setter is never called anywhere, so it was permanently `false`. 2. The **"connection expired"** banner *inside* the `connectedProject` branch was unreachable: `VercelSettingsPresenter` only populates `connectedProject` on its success exit, which hardcodes `authInvalid: false`, while both `authInvalid: true` exits return `connectedProject: undefined`. Removing them makes the surrounding `!showAuthInvalid` guards vacuous, and the `onboardingData?.authInvalid` disjunct redundant — the loader already folds onboarding auth state into `authInvalid` before it reaches the component. **No behaviour change.** An org with a connected project and an expired token still gets the banner, from the branch below (untouched). ## `fix`: gate Staging settings on plans without a Staging environment (TRI-12646) The ticket's premise was inverted, and I've corrected it there. In Git settings, **Preview** is the row that's correctly gated; **Staging** is the one with no gate at all: - Preview swaps its switch for an Upgrade button, and `projectSettings.server.ts` neutralises a forged `previewDeploymentsEnabled=on`. - Staging was a plain always-editable `Input`, and `validateStagingBranch` only checked the branch existed on GitHub. An org without a staging environment could type a tracking branch, hit Save, get a success toast, and have it silently do nothing. Staging and Preview environments are created together for projects on a plan that includes them, so gating one and not the other was an oversight. The Staging row now mirrors the Preview row. Server-side it ignores the submitted branch when there's no staging environment, but **preserves the stored branch rather than clearing it** — deliberately different from the Preview handling. Forcing a boolean off is harmless; forcing a *string* off would wipe a tracking branch the org had already configured the first time they saved after losing the environment. The Vercel write path had the same gap: `update-config` / `complete-onboarding` / `update-env-mapping` never re-derived available env slugs server-side, so `["stg","preview"]` could be persisted for a project with neither environment, and `createDefaultVercelIntegrationData` turned preview on unconditionally. Both now filter against the project's actual environments, via a pure `restrictConfigToAvailableEnvSlugs` helper that only touches keys present on the input. ## `fix`: show build settings when the GitHub app is disabled (TRI-13488) The page wrapped Git settings, the Vercel section **and** build settings in one `githubAppEnabled` guard, so with the GitHub app off it rendered an empty container. The Vercel section genuinely depends on GitHub — it can't sync environment variables or link deployments without a connected repo — so it stays gated. Build settings don't: they also apply to CLI deploys run with `--native-build-server`, exactly as the section's own description states. They now render regardless. ## `fix`: stop the Vercel onboarding modal spinning forever (TRI-13488) `computeInitialState` starts in `loading-projects` whenever the org has a Vercel integration but no onboarding data yet, and the effect that escapes it waits for `availableProjects !== undefined`. When `getOnboardingData` returns `null` — it does that on any thrown error, and when the org integration row is missing — nothing ever arrives. The empty-array case self-resolves (`[] !== undefined`), so this is specifically the null case. The route can tell "still loading" from "loaded nothing" because its fetcher always requests `?vercelOnboarding=true`; it now passes that down and the modal explains the failure with a retry and a link to check the integration's access on Vercel. ## `fix`: match staging and preview environments consistently (TRI-13488) The four places that ask "does this project have a staging / preview environment?" disagreed. `VercelSettingsPresenter` matched on type with no parent filter, so any preview *branch* row satisfied it — branches are `PREVIEW` rows too. `GitHubSettingsPresenter` and `ProjectSettingsService` matched on slug instead. Slug is the weaker key: it's derived at creation time and legacy rows can carry something else, which is why `memberDevelopmentEnvironmentWhere` deliberately avoids it. All four now match on `type` plus `parentEnvironmentId: null`, which excludes branches without depending on the slug being canonical. ## `fix`: explain when no Vercel environment can be mapped to Staging (TRI-13488) Reported while reviewing the branch. The Staging build settings show *"Set a Vercel environment for Staging first."* whenever the project has a staging environment and no mapping — but the control that sets the mapping only rendered when the Vercel project had at least one custom environment: ``` hint: hasStagingEnvironment && !configValues.vercelStagingEnvironment control: hasStagingEnvironment && customEnvironments.length > 0 ``` So a Vercel project with no custom environments, or one whose custom environments failed to fetch (the presenter swallows that error to `[]`), got an instruction with nothing to act on. Both conditions predate this PR. The mapping row now always renders alongside the hint and explains what to do when there's nothing to choose from, and the build-settings hint says the same thing. ## `chore`: remove the remaining dead code (TRI-13488) - The `"installing"` `OnboardingState` is unproducible — no `setState` call yields it — so its redirect effect, switch arm, `isLoadingState` conjunct and the `vercelAppInstallPath` import it was the only user of are all dead. - `(state as string) !== "completed"` sits in a branch where TypeScript has already narrowed `"completed"` out; the cast is what let it compile. - `hideSectionToggles` was only ever passed alongside `layout="settings"` but only read inside `layout="card"` blocks, so it could never take effect. Removed the prop entirely. - Unused bindings and the helpers only they referenced: `envSlugLabel`, `_formatSelectedEnvs`, `_CompleteOnboardingForm`, `_handleFinishOnboarding`, and the rest. No behaviour change in that commit. ## Not included The three overlapping modal-open effects in `settings.integrations/route.tsx` are left alone — they're defensive against a close-then-reopen race, and untangling them is a behavioural risk with no user-visible payoff. ## Verification `pnpm run typecheck --filter webapp`, `pnpm run lint` and `pnpm run knip` are clean. New `apps/webapp/test/vercelIntegrationConfig.test.ts` covers the slug restriction and the default-config seeding (both pure functions); 39 tests pass across it and the three existing Vercel/project-settings files. The new `projectId` + `slug` query is served by the existing `@@unique([projectId, slug, orgMemberId])` prefix — same access pattern as the preview check it mirrors. refs TRI-12645, TRI-12646, TRI-13488 --- .server-changes/vercel-staging-gating.md | 6 + .../integrations/VercelBuildSettings.tsx | 6 +- .../integrations/VercelOnboardingModal.tsx | 69 +++--- .../v3/GitHubSettingsPresenter.server.ts | 43 +++- .../v3/VercelSettingsPresenter.server.ts | 2 + .../route.tsx | 59 +++-- ...cts.$projectParam.env.$envParam.github.tsx | 53 +++-- ...cts.$projectParam.env.$envParam.vercel.tsx | 220 +++++++----------- .../app/services/projectSettings.server.ts | 53 +++-- .../app/services/vercelIntegration.server.ts | 42 +++- .../vercel/vercelProjectIntegrationSchema.ts | 29 ++- .../test/vercelIntegrationConfig.test.ts | 102 ++++++++ 12 files changed, 435 insertions(+), 249 deletions(-) create mode 100644 .server-changes/vercel-staging-gating.md create mode 100644 apps/webapp/test/vercelIntegrationConfig.test.ts diff --git a/.server-changes/vercel-staging-gating.md b/.server-changes/vercel-staging-gating.md new file mode 100644 index 00000000000..c0000314efd --- /dev/null +++ b/.server-changes/vercel-staging-gating.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The Staging branch setting now shows an upgrade prompt on plans that don't include a Staging environment, instead of looking editable and then silently doing nothing when saved. diff --git a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx index e3be9a4f90e..56bc785368c 100644 --- a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx +++ b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx @@ -49,7 +49,6 @@ type BuildSettingsFieldsProps = { * the pin status is unknown — distinct from "not set". */ currentTriggerVersionFetchFailed?: boolean; /** Hide the section-level master toggles for "Pull env vars" and "Discover new env vars". */ - hideSectionToggles?: boolean; showAtomicDeployments?: boolean; layout?: "settings" | "card"; }; @@ -68,7 +67,6 @@ export function BuildSettingsFields({ onAutoPromoteChange, currentTriggerVersion, currentTriggerVersionFetchFailed, - hideSectionToggles, showAtomicDeployments = true, layout = "card", }: BuildSettingsFieldsProps) { @@ -222,7 +220,7 @@ export function BuildSettingsFields({
- {!hideSectionToggles && availableEnvSlugs.length > 1 && ( + {availableEnvSlugs.length > 1 && (
- {!hideSectionToggles && availableEnvSlugs.length > 1 && ( + {availableEnvSlugs.length > 1 && ( void; vercelManageAccessUrl?: string; }) { const { capture, startSessionRecording } = usePostHogTracking(); - const navigation = useNavigation(); const fetcher = useTypedFetcher(); const envMappingFetcher = useFetcher(); const completeOnboardingFetcher = useFetcher(); - const { Form: _CompleteOnboardingForm } = completeOnboardingFetcher; const [searchParams] = useSearchParams(); const origin = searchParams.get("origin"); const fromMarketplaceContext = origin === "marketplace"; @@ -130,7 +127,6 @@ export function VercelOnboardingModal({ () => onboardingData?.availableProjects ?? [], [onboardingData?.availableProjects] ); - const _hasProjectSelected = onboardingData?.hasProjectSelected ?? false; const customEnvironments = useMemo( () => onboardingData?.customEnvironments ?? [], [onboardingData?.customEnvironments] @@ -224,10 +220,6 @@ export function VercelOnboardingModal({ environmentId: string; displayName: string; } | null>(null); - const _availableEnvSlugsForOnboarding = getAvailableEnvSlugs( - hasStagingEnvironment, - hasPreviewEnvironment - ); const availableEnvSlugsForOnboardingBuildSettings = getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment, hasPreviewEnvironment @@ -375,7 +367,6 @@ export function VercelOnboardingModal({ } break; - case "installing": case "project-selection": case "env-mapping": case "env-var-sync": @@ -459,8 +450,6 @@ export function VercelOnboardingModal({ const overlappingEnvVarsCount = enabledEnvVars.filter((v) => existingVars[v.key]).length; - const _isSubmitting = navigation.state === "submitting" || navigation.state === "loading"; - const actionUrl = vercelResourcePath(organizationSlug, projectSlug, environmentSlug); const handleToggleEnvVar = useCallback((key: string, enabled: boolean) => { @@ -634,19 +623,6 @@ export function VercelOnboardingModal({ gitHubAppInstallations.length, ]); - const _handleFinishOnboarding = useCallback( - (e: React.FormEvent) => { - e.preventDefault(); - const form = e.currentTarget; - const formData = new FormData(form); - completeOnboardingFetcher.submit(formData, { - method: "post", - action: actionUrl, - }); - }, - [completeOnboardingFetcher, actionUrl] - ); - useEffect(() => { if ( completeOnboardingFetcher.data && @@ -698,13 +674,6 @@ export function VercelOnboardingModal({ } }, [state, onClose, trackOnboarding, isGitHubConnectedForOnboarding]); - useEffect(() => { - if (state === "installing") { - const installUrl = vercelAppInstallPath(organizationSlug, projectSlug); - window.location.href = installUrl; - } - }, [state, organizationSlug, projectSlug]); - useEffect(() => { if ( envMappingFetcher.data && @@ -749,7 +718,6 @@ export function VercelOnboardingModal({ state === "loading-projects" || state === "loading-env-mapping" || state === "loading-env-vars" || - state === "installing" || (state === "idle" && !onboardingData); if (isLoadingState) { @@ -758,9 +726,7 @@ export function VercelOnboardingModal({ open={isOpen} onOpenChange={(open) => { if (!open && !fromMarketplaceContext) { - if ((state as string) !== "completed") { - trackOnboarding("vercel onboarding abandoned"); - } + trackOnboarding("vercel onboarding abandoned"); onClose(); } }} @@ -772,9 +738,30 @@ export function VercelOnboardingModal({ Set up Vercel Integration
-
- -
+ {onboardingDataUnavailable ? ( +
+ + We couldn't load your Vercel projects. The integration may have been removed or lost + access to this organization on Vercel. + +
+ {onDataReload && ( + + )} + {vercelManageAccessUrl && ( + + Manage access on Vercel + + )} +
+
+ ) : ( +
+ +
+ )} ); diff --git a/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts b/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts index 53bd034f249..162e44f8a88 100644 --- a/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/GitHubSettingsPresenter.server.ts @@ -19,6 +19,7 @@ export class GitHubSettingsPresenter extends BasePresenter { connectedRepository: undefined, installations: undefined, isPreviewEnvironmentEnabled: undefined, + isStagingEnvironmentEnabled: undefined, }); } @@ -114,7 +115,8 @@ export class GitHubSettingsPresenter extends BasePresenter { }, where: { projectId: projectId, - slug: "preview", + type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ @@ -123,15 +125,42 @@ export class GitHubSettingsPresenter extends BasePresenter { }) ).map((previewEnvironment) => previewEnvironment !== null); + const isStagingEnvironmentEnabled = () => + fromPromise( + (this._replica as PrismaClient).runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId: projectId, + type: "STAGING", + parentEnvironmentId: null, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((stagingEnvironment) => stagingEnvironment !== null); + return ResultAsync.combine([ isPreviewEnvironmentEnabled(), + isStagingEnvironmentEnabled(), findConnectedGithubRepository(), listGithubAppInstallations(), - ]).map(([isPreviewEnvironmentEnabled, connectedGithubRepository, githubAppInstallations]) => ({ - enabled: true, - connectedRepository: connectedGithubRepository, - installations: githubAppInstallations, - isPreviewEnvironmentEnabled, - })); + ]).map( + ([ + isPreviewEnvironmentEnabled, + isStagingEnvironmentEnabled, + connectedGithubRepository, + githubAppInstallations, + ]) => ({ + enabled: true, + connectedRepository: connectedGithubRepository, + installations: githubAppInstallations, + isPreviewEnvironmentEnabled, + isStagingEnvironmentEnabled, + }) + ); } } diff --git a/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts index 10a46c01b3a..841c929d141 100644 --- a/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts @@ -182,6 +182,7 @@ export class VercelSettingsPresenter extends BasePresenter { where: { projectId, type: "STAGING", + parentEnvironmentId: null, }, }), (error) => ({ @@ -199,6 +200,7 @@ export class VercelSettingsPresenter extends BasePresenter { where: { projectId, type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings.integrations/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings.integrations/route.tsx index 864cc300fa4..37ad3d51681 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings.integrations/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings.integrations/route.tsx @@ -59,8 +59,9 @@ export const loader = dashboardLoader( async ({ params, user, ability }) => { const { projectParam, organizationSlug } = params; + const canManageBuildSettings = ability.can("write", { type: "github" }); const canManageIntegrations = - ability.can("write", { type: "github" }) || ability.can("write", { type: "vercel" }); + canManageBuildSettings || ability.can("write", { type: "vercel" }); if (!canManageIntegrations) { throwPermissionDenied("With your current role, you can't manage integrations."); @@ -102,6 +103,7 @@ export const loader = dashboardLoader( githubAppEnabled: gitHubApp.enabled, buildSettings, vercelIntegrationEnabled: OrgIntegrationRepository.isVercelSupported, + canManageBuildSettings, }); } ); @@ -208,7 +210,7 @@ export const action = dashboardAction( ); export default function IntegrationsSettingsPage() { - const { githubAppEnabled, buildSettings, vercelIntegrationEnabled } = + const { githubAppEnabled, buildSettings, vercelIntegrationEnabled, canManageBuildSettings } = useTypedLoaderData(); const project = useProject(); const organization = useOrganization(); @@ -223,6 +225,8 @@ export default function IntegrationsSettingsPage() { const loadVercelOnboarding = vercelFetcher.load; const onboardingData = vercelFetcher.data?.onboardingData ?? null; const hasVercelFetcherData = vercelFetcher.data !== undefined; + const onboardingDataUnavailable = + hasVercelFetcherData && vercelFetcher.state === "idle" && onboardingData === null; const vercelOnboardingPath = `${vercelResourcePath( organization.slug, project.slug, @@ -375,24 +379,27 @@ export default function IntegrationsSettingsPage() { /> )} - - - - Applies to deployments triggered from GitHub, and CLI deployments run with the{" "} - - --native-build-server - {" "} - flag. - - } - /> - - )} + + + + Applies to deployments triggered from GitHub, and CLI deployments run with the{" "} + + --native-build-server + {" "} + flag. + + } + /> + + {/* Vercel Onboarding Modal */} @@ -407,6 +414,7 @@ export default function IntegrationsSettingsPage() { hasStagingEnvironment={vercelFetcher.data?.hasStagingEnvironment ?? false} hasPreviewEnvironment={vercelFetcher.data?.hasPreviewEnvironment ?? false} hasOrgIntegration={vercelFetcher.data?.hasOrgIntegration ?? false} + onboardingDataUnavailable={onboardingDataUnavailable} nextUrl={nextUrl ?? undefined} vercelManageAccessUrl={vercelFetcher.data?.vercelManageAccessUrl} onDataReload={(vercelEnvironmentId) => { @@ -424,7 +432,13 @@ export default function IntegrationsSettingsPage() { ); } -function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings }) { +function BuildSettingsForm({ + buildSettings, + canManageBuildSettings = true, +}: { + buildSettings: BuildSettings; + canManageBuildSettings?: boolean; +}) { const lastSubmission = useActionData() as any; const navigation = useNavigation(); @@ -572,7 +586,12 @@ function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings }) name="action" value="update-build-settings" variant="secondary/small" - disabled={isBuildSettingsLoading || !hasBuildSettingsChanges} + disabled={isBuildSettingsLoading || !hasBuildSettingsChanges || !canManageBuildSettings} + tooltip={ + canManageBuildSettings + ? undefined + : "You don't have permission to manage build settings" + } LeadingIcon={isBuildSettingsLoading ? SpinnerWhite : undefined} > Save diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx index 4421b76daf1..fba94b80bba 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx @@ -779,6 +779,7 @@ function GitHubSettingsRows({ export function ConnectedGitHubRepoForm({ connectedGitHubRepo, previewEnvironmentEnabled, + stagingEnvironmentEnabled, organizationSlug, projectSlug, environmentSlug, @@ -788,6 +789,7 @@ export function ConnectedGitHubRepoForm({ }: { connectedGitHubRepo: ConnectedGitHubRepo; previewEnvironmentEnabled?: boolean; + stagingEnvironmentEnabled?: boolean; organizationSlug: string; projectSlug: string; environmentSlug: string; @@ -956,24 +958,42 @@ export function ConnectedGitHubRepoForm({ { - setGitSettingsValues((prev) => ({ - ...prev, - stagingBranch: e.target.value, - })); - }} - /> + stagingEnvironmentEnabled ? ( + { + setGitSettingsValues((prev) => ({ + ...prev, + stagingBranch: e.target.value, + })); + }} + /> + ) : ( + + Upgrade + + ) } > - + void; @@ -698,19 +695,6 @@ function VercelGitHubWarning() { ); } -function envSlugLabel(slug: EnvSlug): string { - switch (slug) { - case "prod": - return "Production"; - case "stg": - return "Staging"; - case "preview": - return "Preview"; - case "dev": - return "Development"; - } -} - function ConnectedVercelProjectForm({ connectedProject, hasStagingEnvironment, @@ -774,7 +758,7 @@ function ConnectedVercelProjectForm({ stagingEnvChanged || autoPromoteChanged; - const [configForm, _fields] = useForm({ + const [configForm] = useForm({ id: "update-vercel-config", lastResult: lastSubmission, shouldRevalidate: "onSubmit", @@ -833,26 +817,22 @@ function ConnectedVercelProjectForm({ const actionUrl = vercelResourcePath(organizationSlug, projectSlug, environmentSlug); - const availableEnvSlugs = getAvailableEnvSlugs(hasStagingEnvironment, hasPreviewEnvironment); const availableEnvSlugsForBuildSettings = getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment, hasPreviewEnvironment ); + const hasVercelCustomEnvironments = customEnvironments.length > 0; + const disabledEnvSlugsForBuildSettings: Partial> | undefined = hasStagingEnvironment && !configValues.vercelStagingEnvironment - ? { stg: "Set a Vercel environment for Staging first." } + ? { + stg: hasVercelCustomEnvironments + ? "Set a Vercel environment for Staging first." + : "Add a custom environment to this project in Vercel to use Staging.", + } : undefined; - const _formatSelectedEnvs = ( - selected: EnvSlug[], - availableSlugs: EnvSlug[] = availableEnvSlugs - ): string => { - if (selected.length === 0) return "None selected"; - if (selected.length === availableSlugs.length) return "All environments"; - return selected.map(envSlugLabel).join(", "); - }; - return ( <> - {/* Staging environment mapping */} - {hasStagingEnvironment && customEnvironments && customEnvironments.length > 0 && ( + {hasStagingEnvironment && ( - { + if (!Array.isArray(value)) { + const env = customEnvironments?.find((e) => e.id === value); + setConfigValues((prev) => { + const next = { + ...prev, + vercelStagingEnvironment: env + ? { environmentId: env.id, displayName: env.slug } + : null, + }; + // When clearing the staging mapping, strip "stg" from build settings + if (!env) { + next.pullEnvVarsBeforeBuild = prev.pullEnvVarsBeforeBuild.filter( + (s) => s !== "stg" + ); + next.discoverEnvVars = prev.discoverEnvVars.filter((s) => s !== "stg"); + } + return next; + }); + } + }} + items={[{ id: "", slug: "None" }, ...customEnvironments]} + variant="secondary/small" + placeholder="Select environment" + dropdownIcon + text={ + configValues.vercelStagingEnvironment ? ( + + ) : ( + "None" + ) } - }} - items={[{ id: "", slug: "None" }, ...customEnvironments]} - variant="secondary/small" - placeholder="Select environment" - dropdownIcon - text={ - configValues.vercelStagingEnvironment ? ( - - ) : ( - "None" - ) - } - > - {[ - - None - , - ...customEnvironments.map((env) => ( - - - - )), - ]} - -
+ > + {[ + + None + , + ...customEnvironments.map((env) => ( + + + + )), + ]} + +
+ ) } /> )} @@ -1041,7 +1029,6 @@ function ConnectedVercelProjectForm({ } currentTriggerVersion={currentTriggerVersion} currentTriggerVersionFetchFailed={currentTriggerVersionFetchFailed} - hideSectionToggles layout="settings" /> @@ -1205,44 +1192,16 @@ function VercelSettingsPanel({ }) { const fetcher = useTypedFetcher(); const { load } = fetcher; - const _location = useLocation(); const data = fetcher.data; - const [hasError, _setHasError] = useState(false); const [hasFetched, setHasFetched] = useState(false); useEffect(() => { - if (!data?.authInvalid && !hasError && !data && !hasFetched) { + if (!data && !hasFetched) { load(vercelResourcePath(organizationSlug, projectSlug, environmentSlug)); // oxlint-disable-next-line react/set-state-in-effect -- This effect intentionally synchronizes route state after an external or lifecycle change. setHasFetched(true); } - }, [ - organizationSlug, - projectSlug, - environmentSlug, - data?.authInvalid, - hasError, - data, - hasFetched, - load, - ]); - - if (hasError) { - return ( -
-
- -
-

Failed to load Vercel settings

-

- There was an error loading the Vercel integration settings. Please refresh the page to - try again. -

-
-
-
- ); - } + }, [organizationSlug, projectSlug, environmentSlug, data, hasFetched, load]); if (fetcher.state === "loading" && !data) { return ( @@ -1258,40 +1217,30 @@ function VercelSettingsPanel({ } const showGitHubWarning = data.connectedProject && !data.isGitHubConnected; - const showAuthInvalid = data.authInvalid || data.onboardingData?.authInvalid; if (data.connectedProject) { return ( <> - {showAuthInvalid && ( - - )} {showGitHubWarning && } - {!showAuthInvalid && } - {!showAuthInvalid && ( - - )} + + ); } - if (showAuthInvalid) { + if (data.authInvalid) { return ( { const installationId = Number(connectedRepo.repository.installation.appInstallationId); + const oldStagingBranch = connectedRepo.branchTracking?.staging?.branch; - return ResultAsync.combine([ - validateProductionBranch({ - installationId, - fullRepoName: connectedRepo.repository.fullName, - oldProductionBranch: connectedRepo.branchTracking?.prod?.branch, - }), - validateStagingBranch({ - installationId, - fullRepoName: connectedRepo.repository.fullName, - oldStagingBranch: connectedRepo.branchTracking?.staging?.branch, - }), - this.isPreviewEnvironmentEnabled(projectId), - ]); + return this.isStagingEnvironmentEnabled(projectId).andThen((stagingEnvironmentEnabled) => + ResultAsync.combine([ + validateProductionBranch({ + installationId, + fullRepoName: connectedRepo.repository.fullName, + oldProductionBranch: connectedRepo.branchTracking?.prod?.branch, + }), + stagingEnvironmentEnabled + ? validateStagingBranch({ + installationId, + fullRepoName: connectedRepo.repository.fullName, + oldStagingBranch, + }) + : okAsync(oldStagingBranch), + this.isPreviewEnvironmentEnabled(projectId), + ]) + ); }) .map(([productionBranch, stagingBranch, previewEnvironmentEnabled]) => ({ productionBranch, @@ -317,7 +322,8 @@ export class ProjectSettingsService { }, where: { projectId: projectId, - slug: "preview", + type: "PREVIEW", + parentEnvironmentId: null, }, }), (error) => ({ @@ -326,4 +332,23 @@ export class ProjectSettingsService { }) ).map((previewEnvironment) => previewEnvironment !== null); } + + private isStagingEnvironmentEnabled(projectId: string) { + return fromPromise( + this.#prismaClient.runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId: projectId, + type: "STAGING", + parentEnvironmentId: null, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((stagingEnvironment) => stagingEnvironment !== null); + } } diff --git a/apps/webapp/app/services/vercelIntegration.server.ts b/apps/webapp/app/services/vercelIntegration.server.ts index 336519af03b..a95bb12b491 100644 --- a/apps/webapp/app/services/vercelIntegration.server.ts +++ b/apps/webapp/app/services/vercelIntegration.server.ts @@ -20,6 +20,8 @@ import { VercelProjectIntegrationDataSchema, envTypeToSlug, createDefaultVercelIntegrationData, + getAvailableEnvSlugs, + restrictConfigToAvailableEnvSlugs, SKEW_PROTECTION_ENV_VAR_KEY, } from "~/v3/vercel/vercelProjectIntegrationSchema"; @@ -129,6 +131,17 @@ export class VercelIntegrationService { .filter((i): i is VercelProjectIntegrationWithProject => i !== null); } + async #getAvailableEnvSlugs(projectId: string): Promise { + const environments = await this.#prismaClient.runtimeEnvironment.findMany({ + where: { projectId, type: { in: ["STAGING", "PREVIEW"] }, parentEnvironmentId: null }, + select: { type: true }, + }); + + const types = new Set(environments.map((environment) => environment.type)); + + return getAvailableEnvSlugs(types.has("STAGING"), types.has("PREVIEW")); + } + async createVercelProjectIntegration(params: { organizationIntegrationId: string; projectId: string; @@ -142,7 +155,8 @@ export class VercelIntegrationService { params.vercelProjectId, params.vercelProjectName, params.vercelTeamId, - params.vercelTeamSlug + params.vercelTeamSlug, + await this.#getAvailableEnvSlugs(params.projectId) ); return this.#prismaClient.organizationProjectIntegration.create({ @@ -183,6 +197,8 @@ export class VercelIntegrationService { () => undefined ); + const availableEnvSlugs = await this.#getAvailableEnvSlugs(params.projectId); + // Use a serializable transaction to prevent duplicate project integrations // from concurrent selectVercelProject calls (read-then-write race condition). const txResult = await $transaction( @@ -236,7 +252,8 @@ export class VercelIntegrationService { params.vercelProjectId, params.vercelProjectName, teamId, - vercelTeamSlug + vercelTeamSlug, + availableEnvSlugs ); const created = await tx.organizationProjectIntegration.create({ @@ -320,7 +337,10 @@ export class VercelIntegrationService { const updatedConfig = { ...existing.parsedIntegrationData.config, - ...configUpdates, + ...restrictConfigToAvailableEnvSlugs( + configUpdates, + await this.#getAvailableEnvSlugs(projectId) + ), }; const updatedData: VercelProjectIntegrationData = { @@ -578,14 +598,20 @@ export class VercelIntegrationService { prod: {}, preview: {}, }; + const availableEnvSlugs = await this.#getAvailableEnvSlugs(projectId); const updatedData: VercelProjectIntegrationData = { ...existing.parsedIntegrationData, config: { ...existing.parsedIntegrationData.config, - pullEnvVarsBeforeBuild: params.pullEnvVarsBeforeBuild ?? null, - atomicBuilds: params.atomicBuilds ?? null, - discoverEnvVars: params.discoverEnvVars ?? null, - vercelStagingEnvironment: params.vercelStagingEnvironment ?? null, + ...restrictConfigToAvailableEnvSlugs( + { + pullEnvVarsBeforeBuild: params.pullEnvVarsBeforeBuild ?? null, + atomicBuilds: params.atomicBuilds ?? null, + discoverEnvVars: params.discoverEnvVars ?? null, + vercelStagingEnvironment: params.vercelStagingEnvironment ?? null, + }, + availableEnvSlugs + ), }, //This is intentionally not updated here, in case of resetting the onboarding it should not override the existing mapping with an empty one syncEnvVarsMapping: existing.parsedIntegrationData.syncEnvVarsMapping, @@ -610,7 +636,7 @@ export class VercelIntegrationService { projectId, vercelProjectId: updatedData.vercelProjectId, teamId, - vercelStagingEnvironment: params.vercelStagingEnvironment, + vercelStagingEnvironment: updatedData.config.vercelStagingEnvironment, syncEnvVarsMapping, orgIntegration, }); diff --git a/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts b/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts index cde9f708163..c428c7956a7 100644 --- a/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts +++ b/apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts @@ -85,13 +85,16 @@ export function createDefaultVercelIntegrationData( vercelProjectId: string, vercelProjectName: string, vercelTeamId: string | null, - vercelTeamSlug?: string + vercelTeamSlug?: string, + availableEnvSlugs: EnvSlug[] = ALL_ENV_SLUGS ): VercelProjectIntegrationData { + const defaultOn = (["prod", "preview"] as EnvSlug[]).filter((s) => availableEnvSlugs.includes(s)); + return { config: { atomicBuilds: [], - pullEnvVarsBeforeBuild: ["prod", "preview"], - discoverEnvVars: ["prod", "preview"], + pullEnvVarsBeforeBuild: defaultOn, + discoverEnvVars: defaultOn, vercelStagingEnvironment: null, autoPromote: true, }, @@ -142,6 +145,26 @@ export function getAvailableEnvSlugs( }); } +export function restrictConfigToAvailableEnvSlugs( + config: Partial, + availableEnvSlugs: EnvSlug[] +): Partial { + const restricted = { ...config }; + + for (const key of ["atomicBuilds", "pullEnvVarsBeforeBuild", "discoverEnvVars"] as const) { + const slugs = restricted[key]; + if (slugs) { + restricted[key] = slugs.filter((slug) => availableEnvSlugs.includes(slug)); + } + } + + if ("vercelStagingEnvironment" in restricted && !availableEnvSlugs.includes("stg")) { + restricted.vercelStagingEnvironment = null; + } + + return restricted; +} + export function getAvailableEnvSlugsForBuildSettings( hasStagingEnvironment: boolean, hasPreviewEnvironment: boolean diff --git a/apps/webapp/test/vercelIntegrationConfig.test.ts b/apps/webapp/test/vercelIntegrationConfig.test.ts new file mode 100644 index 00000000000..696627d70b2 --- /dev/null +++ b/apps/webapp/test/vercelIntegrationConfig.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect } from "vitest"; +import { + createDefaultVercelIntegrationData, + restrictConfigToAvailableEnvSlugs, +} from "../app/v3/vercel/vercelProjectIntegrationSchema"; + +const STAGING_ENV = { environmentId: "env_123", displayName: "Staging" }; + +describe("restrictConfigToAvailableEnvSlugs", () => { + it("drops slugs the project has no environment for", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { + atomicBuilds: ["prod", "stg"], + pullEnvVarsBeforeBuild: ["prod", "preview"], + discoverEnvVars: ["dev", "stg", "preview"], + }, + ["dev", "prod"] + ); + + expect(restricted.atomicBuilds).toEqual(["prod"]); + expect(restricted.pullEnvVarsBeforeBuild).toEqual(["prod"]); + expect(restricted.discoverEnvVars).toEqual(["dev"]); + }); + + it("keeps slugs the project does have", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { atomicBuilds: ["prod", "stg", "preview"] }, + ["dev", "stg", "prod", "preview"] + ); + + expect(restricted.atomicBuilds).toEqual(["prod", "stg", "preview"]); + }); + + it("only touches keys present on the input", () => { + const restricted = restrictConfigToAvailableEnvSlugs({ atomicBuilds: ["stg"] }, ["prod"]); + + expect(restricted).not.toHaveProperty("pullEnvVarsBeforeBuild"); + expect(restricted).not.toHaveProperty("discoverEnvVars"); + expect(restricted).not.toHaveProperty("vercelStagingEnvironment"); + }); + + it("clears the staging environment mapping when staging is unavailable", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { vercelStagingEnvironment: STAGING_ENV }, + ["dev", "prod", "preview"] + ); + + expect(restricted.vercelStagingEnvironment).toBeNull(); + }); + + it("keeps the staging environment mapping when staging is available", () => { + const restricted = restrictConfigToAvailableEnvSlugs( + { vercelStagingEnvironment: STAGING_ENV }, + ["dev", "stg", "prod"] + ); + + expect(restricted.vercelStagingEnvironment).toEqual(STAGING_ENV); + }); + + it("does not mutate the input", () => { + const config = { atomicBuilds: ["prod", "stg"] as const }; + restrictConfigToAvailableEnvSlugs({ atomicBuilds: [...config.atomicBuilds] }, ["prod"]); + + expect(config.atomicBuilds).toEqual(["prod", "stg"]); + }); +}); + +describe("createDefaultVercelIntegrationData", () => { + it("does not enable preview for a project without a preview environment", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "prod", + ]); + + expect(data.config.pullEnvVarsBeforeBuild).toEqual(["prod"]); + expect(data.config.discoverEnvVars).toEqual(["prod"]); + }); + + it("enables preview when the project has a preview environment", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "stg", + "prod", + "preview", + ]); + + expect(data.config.pullEnvVarsBeforeBuild).toEqual(["prod", "preview"]); + expect(data.config.discoverEnvVars).toEqual(["prod", "preview"]); + }); + + it("never turns atomic builds on by default", () => { + const data = createDefaultVercelIntegrationData("prj_1", "My project", null, undefined, [ + "dev", + "stg", + "prod", + "preview", + ]); + + expect(data.config.atomicBuilds).toEqual([]); + expect(data.config.vercelStagingEnvironment).toBeNull(); + }); +}); From 920892bc1186bc751c1b8a38b8ae4da0c9612f62 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:46:48 +0100 Subject: [PATCH 24/28] feat(webapp,run-store): gen-2 shard arms in read-through and idempotency (#4781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gives read-through and idempotency their gen-2 shard arms, so an id that names its own shard is read there and nowhere else. #4764 has landed, so this now targets `main` directly and no longer depends on an unmerged branch. It builds on what that PR supplied: `resolveShard`, `runOpsShardHandles` and the keyed router. TRI-13431 ## What changes **Read-through routes by `resolveShard`, not by the binary residency classifier.** A gen-2 id reads its own shard's replica once and probes no other store. A gen-1 v1 id still reads new only. **Callers now declare `idKind`.** A cuid gives no way to tell a run id from a waitpoint id, and the two must route differently: - a legacy-classified **run** id reads the legacy replica only — there is no cuid run migration, so the new-store probe cannot find it; - a cuid **waitpoint** keeps the new-first pair probe, which is load-bearing because a cuid waitpoint can be co-located with its run on the new store. There is no default, because a default would pick one of those arms silently. The field `runId` is renamed to `id`, since it carried both kinds already. **`ReadThroughResult` carries `found`.** `source` is an open-ended union once shards exist, so a consumer testing found-ness by listing the hit sources reads a gen-2 hit as a miss. One consumer did exactly that. Discriminating on `found` makes that class of bug a compile error rather than something a reviewer has to spot. **Idempotency resolves its client through one shard-keyed map.** Both call sites go through `clientForShardKey`, so they cannot disagree about which store owns an id. An absent key takes an explicit logged branch to the fallback, not a silent legacy default. The `classify` seam is retyped to return a `ShardKey`: `Residency` (`"NEW"`) and the reserved shard keys (`"new"`) differ only by case, and `ShardKey` collapses to `string`, so the compiler would not have caught feeding one into the other. The dead `isMigrated` branch is deleted. Nothing implemented it, and the one production comment recorded that omitting it was deliberate. **`PostgresRunStore._residency` widens to `ShardKey`.** Still unused; the store stays unaware of its siblings. ## Two behaviour fixes found while doing the above **An unconfigured shard key logs and returns not-found instead of throwing.** The waitpoint route takes the id from a URL parameter, and any base32hex core plus `[a-z0-9]` plus `"2"` parses as gen-2. The route turns a throw into a 500, so throwing here would let any authenticated client generate 500s and error logs by guessing shard chars, of which there are 36. An error-logged not-found is neither silent nor a misroute. Throwing stays correct on the router path, where ids are minted rather than received. **The two cross-seam batch hydration sites were gen-2 blind.** `hydrateRunsAcrossSeam` and `ApiBatchResultsPresenter` classified with the binary `ownerEngine`, so a gen-2 run id joined the gen-1 `new` group, missed there, and — classifying dedicated-family — never reached the legacy probe either. The id was dropped from a bulk-action page and from batch results with no error. Both now partition ids by shard key and read each configured shard once. Also: a gen-2 waitpoint that missed its shard replica fell back to the gen-1 new writer, a different database, silently disabling read-your-writes for the freshly minted token that fallback exists to serve. It now falls back to its own shard's writer. ## Merge safety Inert while `RUN_OPS_SHARDS` is unset: the shard maps are empty, so every gen-2 arm is unreachable, and gen-2 minting is not live yet. The one live change is the gen-1 run arm, and it removes work rather than adding it. `RoutingRunStore.findRun` never forwards the caller's client object — it routes by id and reads only the client's presence and replica brand — so `readRunForEvent`'s "new" closure already resolved a legacy-classified run id to the legacy store. The arm removes a duplicated read of the legacy replica. A test pins this, because a future caller passing a raw client and a run id would lose the pre-cutover 27-char case, which is new-resident but classifies legacy. ## Testing 14 tests added, testcontainers throughout, no mocks. 22 affected test files pass; typecheck, lint, format and knip are clean. Both arms were verified by neutralising them and confirming the new tests fail. The batch-results test needed rewriting after that check: the first version passed with the fix neutralised, because it used one container as both the gen-1 new client and the shard replica, so it was not testing what it claimed. Note for review: run testcontainer suites in small batches. Sixteen at once starves Docker and everything times out at 60 seconds. The run-ops legacy-guard baseline is refreshed in its own commit. The baseline is keyed by line number, so partitioning the batch-results read shifted four pre-existing entries and added one. Baselined violations in that file go from four to five, all reads; the new one is the shard read beside two gen-1 reads already there. No changeset and no `.server-changes` entry: a user notices nothing while the flag is unset. --- .../v3/ApiBatchResultsPresenter.server.ts | 63 +++++- ...points.tokens.$waitpointFriendlyId.wait.ts | 14 +- .../concerns/idempotencyKeys.server.ts | 37 ++-- .../idempotencyResidency.server.test.ts | 69 +++++-- .../concerns/idempotencyResidency.server.ts | 52 +++-- ...solveWaitpointThroughReadThrough.server.ts | 29 ++- .../routeBuilders/apiBuilder.server.ts | 33 ++- .../routeBuilders/unroutableId.server.ts | 19 ++ .../app/v3/runEngineHandlersShared.server.ts | 5 +- .../readThrough.server.test.ts | 195 ++++++++++++++++-- .../v3/runOpsMigration/readThrough.server.ts | 122 +++++++---- .../shardHandles.server.test.ts | 37 ++++ .../v3/runOpsMigration/shardHandles.server.ts | 46 +++++ .../v3/runOpsMigration/track1-baseline.json | 25 ++- .../waitpointTokenResolve.server.test.ts | 5 +- ...lkActionV2.batchReadThrough.server.test.ts | 107 ++++++++++ .../BulkActionV2.batchReadThrough.server.ts | 55 +++-- ...atchResultsPresenter.dedicatedSeam.test.ts | 145 ++++++++++++- .../test/readRunForEvent.replicaLag.test.ts | 72 +++++++ ...ointThroughReadThrough.readthrough.test.ts | 103 ++++++++- apps/webapp/test/unroutableIdStatus.test.ts | 40 ++++ .../run-store/src/PostgresRunStore.ts | 3 +- .../src/runOpsStore.shardMap.test.ts | 80 ++++++- .../run-store/src/runOpsStore.ts | 36 +++- 24 files changed, 1225 insertions(+), 167 deletions(-) create mode 100644 apps/webapp/app/services/routeBuilders/unroutableId.server.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts create mode 100644 apps/webapp/test/unroutableIdStatus.test.ts diff --git a/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts index 67ef45ebd27..abe04948083 100644 --- a/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts @@ -1,5 +1,5 @@ import type { BatchTaskRunExecutionResult } from "@trigger.dev/core/v3"; -import { ownerEngine } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { $replica, type PrismaClientOrTransaction, @@ -13,6 +13,8 @@ import { runStore as defaultRunStore } from "~/v3/runStore.server"; import { BasePresenter } from "./basePresenter.server"; import { boundedIn } from "@trigger.dev/database"; +import { runOpsShardReplicas } from "~/v3/runOpsMigration/shardHandles.server"; +import { logger } from "~/services/logger.server"; /** * Run-ops read-through wiring. All optional; absent (or `splitEnabled` falsy) collapses `call` to * passthrough. `legacyReplica` is a READ REPLICA handle only — there is NO legacy-primary field. @@ -21,6 +23,8 @@ type ApiBatchResultsReadThroughDeps = { splitEnabled?: boolean; newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; + /** Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) keeps today's behaviour. */ + shardReplicas?: ReadonlyMap; isPastRetention?: (runId: string) => boolean; }; @@ -181,16 +185,57 @@ export class ApiBatchResultsPresenter extends BasePresenter { const taskRunIds = batchRun.items.map((item) => item.taskRunId); - const newRows = (await newClient.taskRun.findMany({ - where: { id: { in: boundedIn(taskRunIds) } }, - select: memberRunSelect, - })) as TaskRunWithAttempts[]; + // A gen-2 id is directly routable to its own shard, so it must not join the gen-1 read: + // it would miss there, and (being dedicated-family) never reach the legacy probe either. + const shardReplicas = this.readThrough?.shardReplicas ?? runOpsShardReplicas; + const genOneIds: string[] = []; + const idsByShard = new Map(); + for (const id of taskRunIds) { + const shardKey = resolveShard(id); + if (shardKey === "new" || shardKey === "legacy") { + genOneIds.push(id); + } else if (shardReplicas.has(shardKey)) { + const group = idsByShard.get(shardKey); + group ? group.push(id) : idsByShard.set(shardKey, [id]); + } else { + // Not routable and not a gen-1 shape. A gen-1 store is the wrong database, and a + // dedicated-family id never reaches the legacy probe, so falling back there would + // drop the member silently. Drop it loudly instead. + logger.error("ApiBatchResultsPresenter: gen-2 member on an unconfigured shard key", { + runId: id, + shardKey, + configured: [...shardReplicas.keys()], + }); + } + } + + const newRows = ( + genOneIds.length > 0 + ? ((await newClient.taskRun.findMany({ + where: { id: { in: boundedIn(genOneIds) } }, + select: memberRunSelect, + })) as TaskRunWithAttempts[]) + : [] + ).concat( + ( + await Promise.all( + [...idsByShard.entries()].map( + async ([shardKey, ids]) => + (await shardReplicas.get(shardKey)!.taskRun.findMany({ + where: { id: { in: boundedIn(ids) } }, + select: memberRunSelect, + })) as TaskRunWithAttempts[] + ) + ) + ).flat() + ); const runsById = new Map(newRows.map((run) => [run.id, run])); - // A run-ops id can only live on NEW, so only misses that AREN'T run-ops-shaped are candidates - // for the legacy probe — mirrors readThroughRun's per-id "NEW residency skips legacy" rule. - const legacyCandidateIds = taskRunIds.filter( - (id) => !runsById.has(id) && ownerEngine(id) !== "NEW" + // A dedicated-family id (gen-1 v1 or gen-2) can only live on its own store, so only + // misses that AREN'T dedicated-shaped are candidates for the legacy probe — mirrors + // readThroughRun's per-id "dedicated residency skips legacy" rule. + const legacyCandidateIds = genOneIds.filter( + (id) => !runsById.has(id) && resolveShard(id) === "legacy" ); if (legacyCandidateIds.length > 0) { const legacyRows = (await legacyReplica.taskRun.findMany({ diff --git a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts index ea1ebab0679..566ffc05876 100644 --- a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts +++ b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts @@ -38,7 +38,14 @@ const { action } = createActionApiRoute( }); if (!waitpoint) { - throw json({ error: "Waitpoint not found" }, { status: 404 }); + // Retryable: a miss here can be replica lag. resolveWaitpointThroughReadThrough + // deliberately does not read the legacy primary, so it relies on the caller retrying. + // A plain 404 is not retried by the SDK, which would turn a transient miss into a + // permanent failure. + throw json( + { error: "Waitpoint not found" }, + { status: 404, headers: { "x-should-retry": "true" } } + ); } const _result = await engine.blockRunWithWaitpoint({ @@ -55,6 +62,11 @@ const { action } = createActionApiRoute( { status: 200 } ); } catch (error) { + // A Response thrown inside the try is a deliberate status (the 404 above), not a + // failure. Re-throw it untouched, or every intentional 4xx here becomes a 500. + if (error instanceof Response) { + throw error; + } logger.error("Failed to wait for waitpoint", { runId, waitpointId, error }); throw json({ error: "Failed to wait for waitpoint token" }, { status: 500 }); } diff --git a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts index f6696865e94..dfa2d4f5845 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts @@ -1,4 +1,4 @@ -import { ownerEngine, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, RunId, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction, TaskRun, Waitpoint } from "@trigger.dev/database"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; @@ -13,9 +13,10 @@ import { computeClaimTtlSeconds } from "~/v3/mollifier/claimTtl"; import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server"; import { runStore } from "~/v3/runStore.server"; import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server"; +import { runOpsShardWriters } from "~/v3/runOpsMigration/shardHandles.server"; import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server"; import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; -import { resolveIdempotencyDedupClient } from "./idempotencyResidency.server"; +import { clientForShardKey, resolveIdempotencyDedupClient } from "./idempotencyResidency.server"; import type { TraceEventConcern, TriggerTaskRequest } from "../types"; // In-memory per-org mollifier-enabled check, shared with `evaluateGate` @@ -32,6 +33,16 @@ const resolveOrgMollifierFlag = makeResolveMollifierFlag(); // PG's unique index as the backstop. const MAX_CLEARED_WINNER_REACQUIRES = 5; +// The store that owns a shard key. A function, not a map: the handles are module constants and +// `runOpsShardWriters` is already keyed, so a second structure would add an allocation and, if +// memoised, mutable module state. Reading them lazily also keeps this module importable by +// triggerTask under a `~/db.server` mock that omits them. +function idempotencyClientFor(shardKey: ShardKey): PrismaClientOrTransaction | undefined { + if (shardKey === "legacy") return runOpsLegacyPrisma; + if (shardKey === "new") return runOpsNewPrisma; + return runOpsShardWriters.get(shardKey); +} + // Claim ownership context returned to the caller when the // IdempotencyKeyConcern won a pre-gate claim. Caller MUST publish the // winning runId on pipeline success (`publishClaim`) or release the @@ -172,12 +183,9 @@ export class IdempotencyKeyConcern { { isSplitEnabled, fallbackClient: this.prisma, - newClient: runOpsNewPrisma, - legacyClient: runOpsLegacyPrisma, + clientFor: idempotencyClientFor, resolveMintKind: resolveRunIdMintKind, - // `isMigrated` is intentionally omitted: until a child of a swept - // legacy-id parent can be born on the new DB, the swept-marker override - // would never change the answer, so a child routes by parent id-shape. + logger, } ); @@ -640,12 +648,15 @@ export class IdempotencyKeyConcern { } catch { return null; } - let client: PrismaClientOrTransaction; - try { - client = ownerEngine(internalId) === "NEW" ? runOpsNewPrisma : runOpsLegacyPrisma; - } catch { - client = this.prisma; - } + // The routing store routes by id and never forwards this object, so its identity only + // signals read-your-writes. Resolving it through the shard map keeps the two idempotency + // call sites in agreement and stops this reading as gen-2-unaware. + const client = clientForShardKey( + resolveShard(internalId), + idempotencyClientFor, + this.prisma, + logger + ); return runStore.findRun( { id: internalId, runtimeEnvironmentId: environmentId }, { include: { associatedWaitpoint: true } }, diff --git a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts index 39b806a0f71..976a4ffd784 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { RunId } from "@trigger.dev/core/v3/isomorphic"; import { + clientForShardKey, resolveIdempotencyDedupClient, type ResolveIdempotencyClientDeps, } from "./idempotencyResidency.server"; @@ -9,20 +10,30 @@ import { const FALLBACK = { __tag: "fallback" } as never; const NEW_CLIENT = { __tag: "new" } as never; const LEGACY_CLIENT = { __tag: "legacy" } as never; +const SHARD_A_CLIENT = { __tag: "shard-a" } as never; + +function clientMap() { + return new Map([ + ["new", NEW_CLIENT], + ["legacy", LEGACY_CLIENT], + ["a", SHARD_A_CLIENT], + ]); +} function makeDeps(over: Partial): ResolveIdempotencyClientDeps { return { isSplitEnabled: async () => true, fallbackClient: FALLBACK, - newClient: NEW_CLIENT, - legacyClient: LEGACY_CLIENT, + clientFor: (key) => clientMap().get(key), resolveMintKind: async () => "runOpsId", + // Kept as an injected seam: the real resolveShard is total, so only an injected + // classifier can exercise the throw-to-fallback arm below. classify: (id) => { - if (id.length === 26 && id[25] === "1") return "NEW"; - if (id.length === 25) return "LEGACY"; + if (id.length === 26 && id[25] === "2") return id[24]!; + if (id.length === 26 && id[25] === "1") return "new"; + if (id.length === 25) return "legacy"; throw new Error(`unclassifiable: ${id.length}`); }, - isMigrated: undefined, ...over, }; } @@ -72,29 +83,51 @@ describe("resolveIdempotencyDedupClient", () => { expect(client).toBe(LEGACY_CLIENT); }); - it("routes a swept (migrated) cuid-parent child to the NEW client", async () => { - const cuidParent = RunId.toFriendlyId("c".repeat(25)); + it("falls back to the fallback client when a present parent id is unclassifiable", async () => { const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: cuidParent }, - makeDeps({ isMigrated: async () => true }) + { environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" }, + makeDeps({}) ); - expect(client).toBe(NEW_CLIENT); + expect(client).toBe(FALLBACK); }); - it("routes a non-migrated cuid-parent child to the LEGACY client even when isMigrated is provided", async () => { - const cuidParent = RunId.toFriendlyId("d".repeat(25)); + it("routes a child to its OWN SHARD client when the parent is a gen-2 id", async () => { + const genTwoParent = RunId.toFriendlyId("e".repeat(24) + "a2"); const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: cuidParent }, - makeDeps({ isMigrated: async () => false }) + { environmentForMint: env, parentRunFriendlyId: genTwoParent }, + makeDeps({ resolveMintKind: async () => "cuid" }) // mint flag must NOT win for a child ); - expect(client).toBe(LEGACY_CLIENT); + expect(client).toBe(SHARD_A_CLIENT); }); - it("falls back to the fallback client when a present parent id is unclassifiable", async () => { + it("falls back and logs when a gen-2 parent names an unconfigured shard key", async () => { + const errors: unknown[] = []; + const genTwoParent = RunId.toFriendlyId("f".repeat(24) + "z2"); const client = await resolveIdempotencyDedupClient( - { environmentForMint: env, parentRunFriendlyId: "run_not-a-valid-length" }, - makeDeps({}) + { environmentForMint: env, parentRunFriendlyId: genTwoParent }, + makeDeps({ logger: { error: (_m, meta) => errors.push(meta) } }) ); expect(client).toBe(FALLBACK); + expect(errors).toHaveLength(1); + }); +}); + +describe("clientForShardKey", () => { + it("selects the same client the map holds for each reserved key and shard key", () => { + const clients = clientMap(); + const clientFor = (key: string) => clients.get(key); + expect(clientForShardKey("new", clientFor, FALLBACK)).toBe(NEW_CLIENT); + expect(clientForShardKey("legacy", clientFor, FALLBACK)).toBe(LEGACY_CLIENT); + expect(clientForShardKey("a", clientFor, FALLBACK)).toBe(SHARD_A_CLIENT); + }); + + it("returns the fallback and logs for a key the map does not hold", () => { + const errors: unknown[] = []; + const map = clientMap(); + const client = clientForShardKey("z", (key) => map.get(key), FALLBACK, { + error: (_m, meta) => errors.push(meta), + }); + expect(client).toBe(FALLBACK); + expect(errors).toHaveLength(1); }); }); diff --git a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts index 86f1435654b..f2a731e61ca 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyResidency.server.ts @@ -1,22 +1,44 @@ -import { ownerEngine, RunId, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, RunId, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction } from "@trigger.dev/database"; type MintKind = "cuid" | "runOpsId"; +type Logger = { error: (message: string, meta?: Record) => void }; + export type ResolveIdempotencyClientDeps = { isSplitEnabled: () => Promise; fallbackClient: PrismaClientOrTransaction; - newClient: PrismaClientOrTransaction; - legacyClient: PrismaClientOrTransaction; + /** The store that owns a shard key: the reserved `legacy`/`new`, or a gen-2 shard. */ + clientFor: (shardKey: ShardKey) => PrismaClientOrTransaction | undefined; resolveMintKind: (environment: { organizationId: string; id: string; orgFeatureFlags?: unknown; }) => Promise; - classify?: (id: string) => Residency; - isMigrated?: (id: string) => Promise; + classify?: (id: string) => ShardKey; + logger?: Logger; }; +/** + * The one place an id becomes a client. `ShardKey` collapses to `string`, so the compiler + * cannot catch a wrong key here — an absent key takes an explicit logged branch to the + * fallback rather than a silent `?? legacy`. The configured set is not repeated in the log: + * boot already prints the shard table. + */ +export function clientForShardKey( + shardKey: ShardKey, + clientFor: (shardKey: ShardKey) => PrismaClientOrTransaction | undefined, + fallback: PrismaClientOrTransaction, + logger?: Logger +): PrismaClientOrTransaction { + const client = clientFor(shardKey); + if (client === undefined) { + logger?.error("idempotency: no client configured for shard key", { shardKey }); + return fallback; + } + return client; +} + export async function resolveIdempotencyDedupClient( args: { environmentForMint: { organizationId: string; id: string; orgFeatureFlags?: unknown }; @@ -28,9 +50,9 @@ export async function resolveIdempotencyDedupClient( return deps.fallbackClient; } - const classify = deps.classify ?? ownerEngine; - const clientFor = (residency: Residency): PrismaClientOrTransaction => - residency === "NEW" ? deps.newClient : deps.legacyClient; + const classify = deps.classify ?? resolveShard; + const clientFor = (shardKey: ShardKey): PrismaClientOrTransaction => + clientForShardKey(shardKey, deps.clientFor, deps.fallbackClient, deps.logger); if (args.parentRunFriendlyId) { let parentInternalId: string; @@ -39,18 +61,18 @@ export async function resolveIdempotencyDedupClient( } catch { return deps.fallbackClient; } - let residency: Residency; + let shardKey: ShardKey; try { - residency = classify(parentInternalId); + shardKey = classify(parentInternalId); } catch { return deps.fallbackClient; } - if (residency === "LEGACY" && deps.isMigrated && (await deps.isMigrated(parentInternalId))) { - return deps.newClient; - } - return clientFor(residency); + return clientFor(shardKey); } + // Mint kind, not an id: there is no shard to decode, so this keeps resolving to the + // gen-1 pair exactly as before. Which shard a gen-2 env mints into is the mint layer's + // decision, and this client is a read-your-writes signal rather than a correctness gate. const kind = await deps.resolveMintKind(args.environmentForMint); - return clientFor(kind === "runOpsId" ? "NEW" : "LEGACY"); + return clientFor(kind === "runOpsId" ? "new" : "legacy"); } diff --git a/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts b/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts index ec5adc13a6c..b1c7a2bd05d 100644 --- a/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts +++ b/apps/webapp/app/runEngine/concerns/resolveWaitpointThroughReadThrough.server.ts @@ -1,3 +1,4 @@ +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaReplicaClient } from "~/db.server"; import { runOpsLegacyReplica as defaultLegacyReplica, @@ -5,12 +6,18 @@ import { runOpsNewReplica as defaultNewClient, runOpsSplitReadEnabled as defaultSplitReadEnabled, } from "~/db.server"; +import { + runOpsShardReplicas as defaultShardReplicas, + runOpsShardWriters as defaultShardWriters, +} from "~/v3/runOpsMigration/shardHandles.server"; import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; type ResolveWaitpointDeps = { newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; newPrimary?: PrismaReplicaClient; + shardReplicas?: ReadonlyMap; + shardWriters?: ReadonlyMap; splitEnabled?: boolean; isPastRetention?: (id: string) => boolean; }; @@ -21,6 +28,8 @@ export type ResolveWaitpointReadThroughDefaults = { newClient: PrismaReplicaClient; legacyReplica: PrismaReplicaClient; newPrimary: PrismaReplicaClient; + shardReplicas: ReadonlyMap; + shardWriters: ReadonlyMap; splitEnabled: boolean; }; @@ -28,6 +37,8 @@ const productionDefaults: ResolveWaitpointReadThroughDefaults = { newClient: defaultNewClient, legacyReplica: defaultLegacyReplica, newPrimary: defaultNewPrimary as unknown as PrismaReplicaClient, + shardReplicas: defaultShardReplicas, + shardWriters: defaultShardWriters as unknown as ReadonlyMap, splitEnabled: defaultSplitReadEnabled, }; @@ -43,7 +54,8 @@ export async function resolveWaitpointThroughReadThrough(opts: { const splitEnabled = opts.deps?.splitEnabled ?? defaults.splitEnabled; const result = await readThroughRun({ - runId: opts.waitpointId, + id: opts.waitpointId, + idKind: "waitpoint", environmentId: opts.environmentId, readNew: (client) => opts.read(client), readLegacy: (replica) => opts.read(replica), @@ -51,22 +63,31 @@ export async function resolveWaitpointThroughReadThrough(opts: { splitEnabled, newClient: opts.deps?.newClient ?? defaults.newClient, legacyReplica: opts.deps?.legacyReplica ?? defaults.legacyReplica, + shardReplicas: opts.deps?.shardReplicas ?? defaults.shardReplicas, isPastRetention: opts.deps?.isPastRetention, }, }); - if (result.source === "new" || result.source === "legacy-replica") { + if (result.found) { return result.value; } // past-retention is an intentional not-found: the token is gone. - if (result.source === "past-retention") { + if (result.reason === "past-retention") { return null; } // Read-your-writes fallback for a token completed immediately after mint, before it replicated: - // re-read from the run-ops PRIMARY only. We deliberately never read the control-plane/legacy + // re-read from the owning store's PRIMARY only. We deliberately never read the control-plane/legacy // primary here (that is the load the replica-only read-through exists to shed), so a legacy-resident // token that misses its replica stays a miss and the caller retries, rather than adding primary load. + const shardKey = resolveShard(opts.waitpointId); + if (shardKey !== "new" && shardKey !== "legacy") { + // A gen-2 token's primary is its OWN shard's writer. The gen-1 new writer is a different + // database, so reading it would miss and silently disable read-your-writes here. + const shardWriter = (opts.deps?.shardWriters ?? defaults.shardWriters).get(shardKey); + return shardWriter ? await opts.read(shardWriter) : null; + } + const fromNewPrimary = await opts.read(opts.deps?.newPrimary ?? defaults.newPrimary); if (fromNewPrimary != null) { return fromNewPrimary; diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index a5a808e3195..1f0004dda1a 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -25,13 +25,14 @@ import { getApiVersion } from "~/api/versions"; import { WORKER_HEADERS } from "@trigger.dev/core/v3/runEngineWorker"; import { ServiceValidationError } from "~/v3/services/common.server"; import { EngineServiceValidationError } from "@internal/run-engine"; +import { unroutableIdResponse } from "./unroutableId.server"; import { tenantContext, tenantContextFromAuthEnvironment } from "~/services/tenantContext.server"; // Client aborts and service-level validation errors aren't bugs — they're // expected at API boundaries. Log them at `warn` so they stay in stdout // without flowing to Sentry via Logger.onError. function logBoundaryError( - message: "Error in loader" | "Error in action", + message: "Error in loader" | "Error in action" | "Unroutable id", error: unknown, url: string ) { @@ -451,6 +452,12 @@ export function createLoaderApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in loader", error, request.url); return await wrapResponse( @@ -722,6 +729,12 @@ export function createLoaderPATApiRoute< if (error instanceof Response) { return await wrapResponse(request, error, corsStrategy !== "none"); } + + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } return await wrapResponse( request, json({ error: "Internal Server Error" }, { status: 500 }), @@ -996,6 +1009,12 @@ export function createActionPATApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); // Typed validation errors map to their own status (default 400); @@ -1346,6 +1365,12 @@ export function createActionApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); return await wrapResponse( @@ -1612,6 +1637,12 @@ export function createMultiMethodApiRoute< return await wrapResponse(request, error, corsStrategy !== "none"); } + const unroutable = unroutableIdResponse(error); + if (unroutable) { + logBoundaryError("Unroutable id", error, request.url); + return await wrapResponse(request, unroutable, corsStrategy !== "none"); + } + logBoundaryError("Error in action", error, request.url); return await wrapResponse( diff --git a/apps/webapp/app/services/routeBuilders/unroutableId.server.ts b/apps/webapp/app/services/routeBuilders/unroutableId.server.ts new file mode 100644 index 00000000000..dcc2d46d7b6 --- /dev/null +++ b/apps/webapp/app/services/routeBuilders/unroutableId.server.ts @@ -0,0 +1,19 @@ +import { json } from "@remix-run/server-runtime"; +import { UnknownShardKey } from "@internal/run-store"; + +/** + * An id naming a shard the topology has no store for cannot be routed, so a read cannot locate + * the row: that is a 404, and matches what an absent gen-1 or cuid id already returns. It must + * not be a 500 — `resolveShard` is pure id-shape, so any base32hex core plus `[a-z0-9]` plus "2" + * parses as gen-2, which lets any caller induce a 5xx, and a 5xx on a read trips canary rollbacks. + * + * The router still throws. Callers log it before returning this, so a genuine misconfiguration — + * a shard key dropped from a config that is meant to be append-only — still alarms. + */ +export function unroutableIdResponse(error: unknown): Response | undefined { + // Explicitly NOT retryable: an id naming an unconfigured shard is not a transient miss, and + // no number of retries makes a topology grow a store. + return error instanceof UnknownShardKey + ? json({ error: "Not Found" }, { status: 404, headers: { "x-should-retry": "false" } }) + : undefined; +} diff --git a/apps/webapp/app/v3/runEngineHandlersShared.server.ts b/apps/webapp/app/v3/runEngineHandlersShared.server.ts index 4ce8cc2de8a..d8999e2332a 100644 --- a/apps/webapp/app/v3/runEngineHandlersShared.server.ts +++ b/apps/webapp/app/v3/runEngineHandlersShared.server.ts @@ -35,7 +35,8 @@ export async function readRunForEvent( deps: EventReadDeps ): Promise | null> { const result = await readThroughRun>({ - runId, + id: runId, + idKind: "run", environmentId, readNew: (client) => deps.store.findRun({ id: runId }, { select }, client), readLegacy: (replica) => deps.store.findRun({ id: runId }, { select }, replica), @@ -47,7 +48,7 @@ export async function readRunForEvent( }, }); - return result.source === "not-found" || result.source === "past-retention" ? null : result.value; + return result.found ? result.value : null; } /** diff --git a/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts b/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts index f7f7c43a530..8a657060ef9 100644 --- a/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts @@ -13,6 +13,21 @@ vi.setConfig({ testTimeout: 60_000 }); // 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency. const LEGACY_RUN_ID = "run_" + "a".repeat(25); const NEW_RUN_ID = "run_" + "b".repeat(24) + "01"; +// 26-char gen-2 body: shard char at index 24, version "2" at index 25. +const SHARD_A_RUN_ID = "run_" + "c".repeat(24) + "a2"; +const SHARD_Z_RUN_ID = "run_" + "c".repeat(24) + "z2"; +const LEGACY_WAITPOINT_ID = "waitpoint_" + "d".repeat(25); + +function throwingClient(label: string) { + return vi.fn(async (): Promise<{ marker: number } | null> => { + throw new Error(`${label} must never be read`); + }); +} + +function collectingLogger() { + const errors: { message: string; meta?: unknown }[] = []; + return { errors, error: (message: string, meta?: unknown) => errors.push({ message, meta }) }; +} // Lightweight real read: a trivial `$queryRaw` that genuinely hits the given container. // `hit` controls whether the read "finds" the run, so we exercise routing without @@ -28,14 +43,7 @@ async function realRead( // A presenter-shaped mapping: both "not-found" and "past-retention" collapse to the // same 404-ish surface, so an old run after termination yields the normal response. function toHttpish(result: ReadThroughResult): { status: number; value?: T } { - switch (result.source) { - case "new": - case "legacy-replica": - return { status: 200, value: result.value }; - case "not-found": - case "past-retention": - return { status: 404 }; - } + return result.found ? { status: 200, value: result.value } : { status: 404 }; } describe("readThroughRun (legacy replica + new DB)", () => { @@ -46,7 +54,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { // read resolving through `legacyReplica` (prisma14) IS the structural guarantee // that the primary is never touched. const result = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, true), @@ -57,7 +66,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("legacy-replica"); + expect(result.found && result.source).toBe("legacy-replica"); expect(toHttpish(result).status).toBe(200); } ); @@ -66,7 +75,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { "post-termination past-retention returns the normal not-found surface", async ({ prisma14, prisma17 }) => { const pastRetentionResult = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, false), // legacy gone / retention elapsed @@ -78,11 +88,14 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(pastRetentionResult.source).toBe("past-retention"); + expect(pastRetentionResult.found === false && pastRetentionResult.reason).toBe( + "past-retention" + ); // A run that is simply absent (not past retention) yields not-found. const notFoundResult = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: (c) => realRead(c, false), readLegacy: (c) => realRead(c, false), @@ -94,7 +107,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(notFoundResult.source).toBe("not-found"); + expect(notFoundResult.found === false && notFoundResult.reason).toBe("not-found"); // Both collapse to the same 404-ish surface. expect(toHttpish(pastRetentionResult).status).toBe(toHttpish(notFoundResult).status); expect(toHttpish(pastRetentionResult).status).toBe(404); @@ -110,7 +123,8 @@ describe("readThroughRun (legacy replica + new DB)", () => { const newRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); const result = await readThroughRun({ - runId: LEGACY_RUN_ID, + id: LEGACY_RUN_ID, + idKind: "run", environmentId: "env_1", readNew: newRead, readLegacy: throwingLegacy, @@ -121,7 +135,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("new"); + expect(result.found && result.source).toBe("new"); expect(newRead).toHaveBeenCalledTimes(1); expect(throwingLegacy).not.toHaveBeenCalled(); } @@ -135,7 +149,152 @@ describe("readThroughRun (legacy replica + new DB)", () => { }); const result = await readThroughRun({ - runId: NEW_RUN_ID, + id: NEW_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: (c) => realRead(c, true), + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("new"); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-2 id reads its OWN shard replica once and probes no other store", + async ({ prisma14, prisma17 }) => { + const throwingNew = throwingClient("the gen-1 new store"); + const throwingLegacy = throwingClient("the legacy replica"); + const shardRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + const result = await readThroughRun({ + id: SHARD_A_RUN_ID, + idKind: "run", + environmentId: "env_1", + // One closure serves both the gen-1 new store and a shard: a shard is the same + // dedicated schema. The throwing clients prove WHICH client it was handed. + readNew: (c) => shardRead(c), + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: throwingNew as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(result.found && result.source).toBe("shard:a"); + expect(shardRead).toHaveBeenCalledTimes(1); + // Identity, not deep equality: a Prisma client is too large to deep-compare. + expect(shardRead.mock.calls[0][0]).toBe(prisma17); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-2 id on an UNCONFIGURED shard key logs an error and returns not-found, never throws", + async ({ prisma14, prisma17 }) => { + const logger = collectingLogger(); + const throwingLegacy = throwingClient("the legacy replica"); + const newRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + // Shard "z" is not configured. A 500 here would be inducible by any caller that + // guesses a shard char, so the layer must degrade rather than throw. + const result = await readThroughRun({ + id: SHARD_Z_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: newRead, + readLegacy: throwingLegacy, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + logger, + }, + }); + + expect(result.found).toBe(false); + expect(result.found === false && result.reason).toBe("not-found"); + expect(logger.errors).toHaveLength(1); + expect(logger.errors[0].meta).toMatchObject({ shardKey: "z", configured: ["a"] }); + // It must not silently fall back onto a gen-1 store. + expect(newRead).not.toHaveBeenCalled(); + expect(throwingLegacy).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "gen-1 RUN id reads the legacy replica only and never probes the new store", + async ({ prisma14 }) => { + const throwingNew = throwingClient("the new store"); + const legacyRead = vi.fn((c: PrismaReplicaClient) => realRead(c, true)); + + const result = await readThroughRun({ + id: LEGACY_RUN_ID, + idKind: "run", + environmentId: "env_1", + readNew: throwingNew, + readLegacy: legacyRead, + deps: { + splitEnabled: true, + newClient: prisma14 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("legacy-replica"); + expect(throwingNew).not.toHaveBeenCalled(); + expect(legacyRead).toHaveBeenCalledTimes(1); + } + ); + + heteroPostgresTest( + "cuid WAITPOINT id keeps the new-FIRST pair probe (frozen: cuid waitpoints co-locate on new)", + async ({ prisma14, prisma17 }) => { + const calls: string[] = []; + const newRead = vi.fn(async (c: PrismaReplicaClient) => { + calls.push("new"); + return realRead(c, false); + }); + const legacyRead = vi.fn(async (c: PrismaReplicaClient) => { + calls.push("legacy"); + return realRead(c, true); + }); + + const result = await readThroughRun({ + id: LEGACY_WAITPOINT_ID, + idKind: "waitpoint", + environmentId: "env_1", + readNew: newRead, + readLegacy: legacyRead, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(result.found && result.source).toBe("legacy-replica"); + expect(calls).toEqual(["new", "legacy"]); + } + ); + + heteroPostgresTest( + "a cuid waitpoint found on the new store returns it without touching legacy", + async ({ prisma14, prisma17 }) => { + const throwingLegacy = throwingClient("the legacy replica"); + + const result = await readThroughRun({ + id: LEGACY_WAITPOINT_ID, + idKind: "waitpoint", environmentId: "env_1", readNew: (c) => realRead(c, true), readLegacy: throwingLegacy, @@ -146,7 +305,7 @@ describe("readThroughRun (legacy replica + new DB)", () => { }, }); - expect(result.source).toBe("new"); + expect(result.found && result.source).toBe("new"); expect(throwingLegacy).not.toHaveBeenCalled(); } ); diff --git a/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts b/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts index f15230ec442..6e1beaae62c 100644 --- a/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/readThrough.server.ts @@ -3,12 +3,18 @@ * (which carries the read load we are shedding). Disabled entirely when isSplitEnabled() * is false (single-DB passthrough). * - * During the retention window, old run-ops rows are served off the legacy read replica. - * Residency is decided purely by id-shape: a run-ops id (NEW) id reads new only, a cuid - * (LEGACY) id reads legacy only. An unclassifiable id falls back to a new-then-legacy - * probe. After termination, past-retention runs return the normal not-found response. - * Patterned on `mollifier/resolveRunForMutation.server.ts` (`?? default` DI), but with - * the legacy-primary/writer fallback deliberately removed: this layer has NO legacy-writer + * Residency is decided purely by id-shape, via `resolveShard`: a gen-2 body names its own + * shard (ONE read there), a gen-1 v1 body reads new only, everything else is legacy and + * routes on `idKind`. + * + * `idKind` is required because a cuid gives no way to tell a run id from a waitpoint id, + * and the two must route differently: a legacy-classified RUN id is legacy-resident (there + * is no cuid run migration), while a cuid WAITPOINT can be co-located with its run on the + * new store, which is what makes the new-first probe load-bearing for it. No default — + * a default would pick one of those arms silently. + * + * Patterned on `mollifier/resolveRunForMutation.server.ts` (`?? default` DI), but with the + * legacy-primary/writer fallback deliberately removed: this layer has NO legacy-writer * handle at all (structural guarantee). */ import type { PrismaReplicaClient } from "~/db.server"; @@ -17,90 +23,118 @@ import { runOpsNewReplica as defaultNewClient, } from "~/db.server"; import { logger as defaultLogger } from "~/services/logger.server"; -import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { isSplitEnabled } from "./splitMode.server"; +import { runOpsShardReplicas } from "./shardHandles.server"; + +type ShardSource = `shard:${string}`; -type ReadThroughSource = "new" | "legacy-replica"; +type ReadThroughSource = "new" | "legacy-replica" | ShardSource; +/** + * `found` carries hit/miss STRUCTURALLY. `source` is open-ended once shards exist, so a + * consumer testing found-ness by listing hit sources reads a gen-2 hit as a miss; + * discriminating on `found` makes that a compile error instead. + */ export type ReadThroughResult = - | { source: ReadThroughSource; value: T } - | { source: "not-found" } - | { source: "past-retention" }; + | { found: true; source: ReadThroughSource; value: T } + | { found: false; reason: "not-found" | "past-retention" }; type ReadThroughDeps = { newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; + /** + * Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) makes the gen-2 arm + * unreachable. Load-bearing only for callers whose closures read a client DIRECTLY: + * `RoutingRunStore` never forwards a caller's client, so for store-backed closures the + * client picked here is only a read-your-writes signal. Not dead weight. + */ + shardReplicas?: ReadonlyMap; /** Resolved boot constant; never `await`ed per-request when supplied. */ splitEnabled?: boolean; - isPastRetention?: (runId: string) => boolean; - logger?: { warn: (m: string, meta?: unknown) => void }; + isPastRetention?: (id: string) => boolean; + logger?: { error: (m: string, meta?: Record) => void }; /** Saturation-signal emit hook: called on each legacy-replica hit. */ - onLegacyReplicaRead?: (runId: string) => void; + onLegacyReplicaRead?: (id: string) => void; }; type ReadThroughRunInput = { - runId: string; + id: string; + idKind: "run" | "waitpoint"; environmentId: string; readNew: (client: PrismaReplicaClient) => Promise; readLegacy: (replica: PrismaReplicaClient) => Promise; deps?: ReadThroughDeps; }; +function hit(source: ReadThroughSource, value: T): ReadThroughResult { + return { found: true, source, value }; +} + +function miss(reason: "not-found" | "past-retention"): ReadThroughResult { + return { found: false, reason }; +} + export async function readThroughRun( input: ReadThroughRunInput ): Promise> { - const { runId, deps } = input; + const { id, idKind, deps } = input; const newClient = deps?.newClient ?? defaultNewClient; const legacyReplica = deps?.legacyReplica ?? defaultLegacyReplica; + const shardReplicas = deps?.shardReplicas ?? runOpsShardReplicas; const logger = deps?.logger ?? defaultLogger; const splitEnabled = deps?.splitEnabled ?? (await isSplitEnabled()); - // Passthrough: single plain read against the one collapsed store. No legacy read, - // no second connection. + // Passthrough: single plain read against the one collapsed store. if (!splitEnabled) { const v = await input.readNew(newClient); - return v != null ? { source: "new", value: v } : { source: "not-found" }; + return v != null ? hit("new", v) : miss("not-found"); } - // Split is on. Classify residency; an unclassifiable id is treated as LEGACY - // (conservative — probe rather than drop a real run). - let residency: "LEGACY" | "NEW"; - try { - residency = ownerEngine(runId); - } catch (e) { - if (e instanceof UnclassifiableRunId) { - logger.warn("readThroughRun: UnclassifiableRunId, treating as LEGACY", { - runId, - valueLength: e.valueLength, + // Total: an unclassifiable id resolves to "legacy" (probe rather than drop a real run). + const shardKey = resolveShard(id); + + if (shardKey !== "new" && shardKey !== "legacy") { + const shardReplica = shardReplicas.get(shardKey); + if (shardReplica === undefined) { + // Deliberately not a throw: this id arrives from the caller (a URL param on the + // waitpoint route) and any base32hex core + [a-z0-9] + "2" parses as gen-2, so a + // throw is a 500 any client can induce. An error-logged not-found is neither silent + // nor a misroute. Throwing stays correct on the router path, where ids are minted. + logger.error("readThroughRun: gen-2 id resolved to an unconfigured shard key", { + id, + shardKey, + configured: [...shardReplicas.keys()], }); - residency = "LEGACY"; - } else { - throw e; + return miss("not-found"); } + // A gen-2 shard is a dedicated-schema store, exactly like `new`, so `readNew` fits. + const v = await input.readNew(shardReplica); + return v != null ? hit(`shard:${shardKey}`, v) : miss("not-found"); } - // A run-ops id can only live on the new DB — skip the legacy replica entirely. - if (residency === "NEW") { + if (shardKey === "new") { const v = await input.readNew(newClient); - return v != null ? { source: "new", value: v } : { source: "not-found" }; + return v != null ? hit("new", v) : miss("not-found"); } - // LEGACY (or unclassifiable→LEGACY) fan-out: new first. - const v = await input.readNew(newClient); - if (v != null) { - return { source: "new", value: v }; + if (idKind === "waitpoint") { + const v = await input.readNew(newClient); + if (v != null) { + return hit("new", v); + } } // Legacy READ REPLICA only — never a legacy writer/primary (no such handle exists). const lv = await input.readLegacy(legacyReplica); if (lv != null) { - deps?.onLegacyReplicaRead?.(runId); - return { source: "legacy-replica", value: lv }; + deps?.onLegacyReplicaRead?.(id); + return hit("legacy-replica", lv); } - if (deps?.isPastRetention?.(runId)) { - return { source: "past-retention" }; + if (deps?.isPastRetention?.(id)) { + return miss("past-retention"); } - return { source: "not-found" }; + return miss("not-found"); } diff --git a/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts new file mode 100644 index 00000000000..b90c1dfe7c4 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { buildShardHandleMaps } from "./shardHandles.server"; + +// Two distinct sentinels per shard: the maps must not cross writer and replica. +function handle(key: string) { + return { + key, + writer: { tag: `${key}-writer` } as never, + replica: { tag: `${key}-replica` } as never, + }; +} + +describe("buildShardHandleMaps", () => { + it("yields empty maps when no shard is configured", () => { + const { replicas, writers } = buildShardHandleMaps([]); + + expect(replicas.size).toBe(0); + expect(writers.size).toBe(0); + }); + + it("keys each shard's replica and writer under its shard char", () => { + const { replicas, writers } = buildShardHandleMaps([handle("a"), handle("b")]); + + expect([...replicas.keys()].sort()).toEqual(["a", "b"]); + expect([...writers.keys()].sort()).toEqual(["a", "b"]); + expect(replicas.get("a")).toEqual({ tag: "a-replica" }); + expect(writers.get("a")).toEqual({ tag: "a-writer" }); + expect(replicas.get("b")).toEqual({ tag: "b-replica" }); + expect(writers.get("b")).toEqual({ tag: "b-writer" }); + }); + + it("never places a writer in the replica map", () => { + const { replicas } = buildShardHandleMaps([handle("a")]); + + expect(replicas.get("a")).not.toEqual({ tag: "a-writer" }); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts new file mode 100644 index 00000000000..cbe827be4dc --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/shardHandles.server.ts @@ -0,0 +1,46 @@ +/** + * Gen-2 shard client handles, keyed by shard char, for the consumers that route by + * `resolveShard` outside the run-store boundary: read-through and the two cross-seam + * batch hydration sites. Both maps are empty unless RUN_OPS_SHARDS is configured, which + * is what keeps every gen-2 arm unreachable today. + */ +import type { PrismaClient } from "@trigger.dev/database"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaReplicaClient } from "~/db.server"; +import { runOpsShardHandles } from "~/db.server"; + +type ShardHandle = { + key: string; + writer: unknown; + replica: unknown; +}; + +export function buildShardHandleMaps(handles: ShardHandle[]): { + replicas: ReadonlyMap; + writers: ReadonlyMap; +} { + const replicas = new Map(); + const writers = new Map(); + for (const handle of handles) { + replicas.set(handle.key, handle.replica as PrismaReplicaClient); + writers.set(handle.key, handle.writer as PrismaClient); + } + return { replicas, writers }; +} + +// A gen-2 shard is the same dedicated subset schema as the gen-1 new store, so these casts +// carry exactly the precedent (and the same residual risk) as `runOpsNewPrisma`'s. +// The try/catch mirrors `runStore.server.ts`'s handle resolution: a minimal `db.server` mock +// does not define this export at all, and accessing an undefined mock export throws. +function resolveShardHandles(): ShardHandle[] { + try { + return runOpsShardHandles ?? []; + } catch { + return []; + } +} + +const maps = buildShardHandleMaps(resolveShardHandles()); + +export const runOpsShardReplicas = maps.replicas; +export const runOpsShardWriters = maps.writers; diff --git a/apps/webapp/app/v3/runOpsMigration/track1-baseline.json b/apps/webapp/app/v3/runOpsMigration/track1-baseline.json index 63d27dbadca..26d038678ea 100644 --- a/apps/webapp/app/v3/runOpsMigration/track1-baseline.json +++ b/apps/webapp/app/v3/runOpsMigration/track1-baseline.json @@ -68,19 +68,19 @@ "WaitpointTag.project" ], "totals": { - "violations": 4, - "detectorI": 4, + "violations": 5, + "detectorI": 5, "detectorII": 0, "detectorIII": 0, "write": 0, - "read": 4, + "read": 5, "files": 1, "legacyAnnotations": 0 }, "violations": [ { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 89, + "line": 93, "model": "BatchTaskRun", "delegate": "batchTaskRun", "callKind": "read", @@ -89,7 +89,7 @@ }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 150, + "line": 154, "model": "BatchTaskRun", "delegate": "batchTaskRun", "callKind": "read", @@ -98,16 +98,25 @@ }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 184, + "line": 214, "model": "TaskRun", "delegate": "taskRun", "callKind": "read", "detector": "i", - "snippet": "const newRows = (await newClient.taskRun.findMany({" + "snippet": "? ((await newClient.taskRun.findMany({" }, { "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", - "line": 196, + "line": 224, + "model": "TaskRun", + "delegate": "taskRun", + "callKind": "read", + "detector": "i", + "snippet": "(await shardReplicas.get(shardKey)!.taskRun.findMany({" + }, + { + "file": "apps/webapp/app/presenters/v3/ApiBatchResultsPresenter.server.ts", + "line": 241, "model": "TaskRun", "delegate": "taskRun", "callKind": "read", diff --git a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts index 9ea849c8058..42dca92e1ce 100644 --- a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts @@ -138,7 +138,8 @@ describe("public wait-token resolution across the split boundary", () => { expect(gated?.id).toBe(waitpointId); const passthrough = await readThroughRun({ - runId: waitpointId, + id: waitpointId, + idKind: "waitpoint", environmentId: environment.id, readNew: (c) => read(c), readLegacy: (r) => read(r), @@ -150,7 +151,7 @@ describe("public wait-token resolution across the split boundary", () => { }); expect(gated).not.toBeNull(); - expect(passthrough.source).toBe("not-found"); + expect(passthrough.found === false && passthrough.reason).toBe("not-found"); } ); }); diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts index 99d4cfd2dd7..779a7234748 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts @@ -13,6 +13,8 @@ vi.setConfig({ testTimeout: 60_000 }); // 25-char cuid body → LEGACY residency. 26-char v1 body (version "1" at index 25) → NEW residency. const LEGACY_RUN_ID = "run_" + "a".repeat(25); const NEW_RUN_ID = "run_" + "b".repeat(24) + "01"; +// 26-char gen-2 body: shard char at index 24, version "2" at index 25. +const SHARD_A_RUN_ID = "run_" + "c".repeat(24) + "a2"; type Row = { id: string }; @@ -90,4 +92,109 @@ describe("hydrateRunsAcrossSeam (PG14 legacy replica + PG17 new)", () => { expect(throwingLegacy).not.toHaveBeenCalled(); } ); + + heteroPostgresTest( + "(c) a gen-2 id hydrates from its OWN shard and is never read from the gen-1 stores", + async ({ prisma14, prisma17 }) => { + // Before the shard arm existed a gen-2 id joined the `new` group, missed, and was + // never legacy-probed either — so it vanished from the page with no error. + const onShardA = new Set([SHARD_A_RUN_ID]); + + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + if ( + ids.includes(SHARD_A_RUN_ID) && + client !== (prisma17 as unknown as PrismaReplicaClient) + ) { + throw new Error("a gen-2 id must only be read on its own shard"); + } + return realReadFiltered(client, ids, onShardA); + }); + const readLegacyReplica = vi.fn( + async (_replica: PrismaReplicaClient, ids: string[]): Promise => { + if (ids.includes(SHARD_A_RUN_ID)) { + throw new Error("a gen-2 id must never reach the legacy probe"); + } + return []; + } + ); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [SHARD_A_RUN_ID], + readNew, + readLegacyReplica, + deps: { + splitEnabled: true, + newClient: prisma14 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(rows.map((r) => r.id)).toEqual([SHARD_A_RUN_ID]); + expect(readLegacyReplica).not.toHaveBeenCalled(); + } + ); + + heteroPostgresTest( + "(d) a mixed gen-1 and gen-2 page hydrates every member", + async ({ prisma14, prisma17 }) => { + const onGenOneNew = new Set([NEW_RUN_ID]); + const onLegacy = new Set([LEGACY_RUN_ID]); + const onShardA = new Set([SHARD_A_RUN_ID]); + + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + const present = ids.includes(SHARD_A_RUN_ID) ? onShardA : onGenOneNew; + return realReadFiltered(client, ids, present); + }); + const readLegacyReplica = vi.fn( + async (replica: PrismaReplicaClient, ids: string[]): Promise => + realReadFiltered(replica, ids, onLegacy) + ); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [NEW_RUN_ID, LEGACY_RUN_ID, SHARD_A_RUN_ID], + readNew, + readLegacyReplica, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", prisma17 as unknown as PrismaReplicaClient]]), + }, + }); + + expect(rows.map((r) => r.id).sort()).toEqual( + [NEW_RUN_ID, LEGACY_RUN_ID, SHARD_A_RUN_ID].sort() + ); + } + ); + + heteroPostgresTest( + "(e) a gen-2 id on an unconfigured shard is dropped with a logged error, not read elsewhere", + async ({ prisma14, prisma17 }) => { + const errors: unknown[] = []; + const readNew = vi.fn(async (client: PrismaReplicaClient, ids: string[]): Promise => { + if (ids.includes(SHARD_A_RUN_ID)) { + throw new Error("an unconfigured gen-2 id must not fall back to a gen-1 store"); + } + return realReadFiltered(client, ids, new Set([NEW_RUN_ID])); + }); + + const rows = await hydrateRunsAcrossSeam({ + runIds: [NEW_RUN_ID, SHARD_A_RUN_ID], + readNew, + readLegacyReplica: async () => [], + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + shardReplicas: new Map(), + logger: { error: (_m, meta) => errors.push(meta) }, + }, + }); + + expect(rows.map((r) => r.id)).toEqual([NEW_RUN_ID]); + expect(errors).toHaveLength(1); + } + ); }); diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts index c7a0dc735e8..bc476cebf31 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.ts @@ -20,7 +20,8 @@ import { runOpsLegacyReplica as defaultLegacyReplica, runOpsNewReplica as defaultNewClient, } from "~/db.server"; -import { ownerEngine, UnclassifiableRunId } from "@trigger.dev/core/v3/isomorphic"; +import { resolveShard, type ShardKey } from "@trigger.dev/core/v3/isomorphic"; +import { runOpsShardReplicas as defaultShardReplicas } from "~/v3/runOpsMigration/shardHandles.server"; type SeamReadDeps = { /** @@ -30,7 +31,9 @@ type SeamReadDeps = { splitEnabled: boolean; newClient?: PrismaReplicaClient; legacyReplica?: PrismaReplicaClient; - logger?: { warn: (m: string, meta?: unknown) => void }; + /** Gen-2 shard replicas by shard char; empty (RUN_OPS_SHARDS unset) keeps today's behaviour. */ + shardReplicas?: ReadonlyMap; + logger?: { error: (m: string, meta?: Record) => void }; }; type HydrateRunsAcrossSeamInput = { @@ -61,28 +64,30 @@ export async function hydrateRunsAcrossSeam(input: HydrateRunsAcrossSeamInput return input.readNew(newClient, runIds); } - // Split is on. Classify residency; unclassifiable → LEGACY (probe rather than drop). + // Split is on. Partition by shard key; `resolveShard` is total, so an unclassifiable id + // resolves to "legacy" (probe rather than drop). A gen-2 id goes to its OWN shard and to + // no other store: it is directly routable, so it joins neither gen-1 group. + const shardReplicas = deps.shardReplicas ?? defaultShardReplicas; const newIds: string[] = []; const legacyCandidateIds: string[] = []; + const idsByShard = new Map(); for (const runId of runIds) { - let residency: "LEGACY" | "NEW"; - try { - residency = ownerEngine(runId); - } catch (e) { - if (e instanceof UnclassifiableRunId) { - deps.logger?.warn("hydrateRunsAcrossSeam: UnclassifiableRunId, treating as LEGACY", { - runId, - valueLength: e.valueLength, - }); - residency = "LEGACY"; - } else { - throw e; - } - } - if (residency === "NEW") { + const shardKey = resolveShard(runId); + if (shardKey === "new") { newIds.push(runId); - } else { + } else if (shardKey === "legacy") { legacyCandidateIds.push(runId); + } else if (shardReplicas.has(shardKey)) { + const group = idsByShard.get(shardKey); + group ? group.push(runId) : idsByShard.set(shardKey, [runId]); + } else { + // Not routable and not a gen-1 shape. Reading a gen-1 store would query the wrong + // database, so the id is dropped from the page — loudly, never silently. + deps.logger?.error("hydrateRunsAcrossSeam: gen-2 id on an unconfigured shard key", { + runId, + shardKey, + configured: [...shardReplicas.keys()], + }); } } @@ -103,6 +108,16 @@ export async function hydrateRunsAcrossSeam(input: HydrateRunsAcrossSeamInput legacyRows = await input.readLegacyReplica(legacyReplica, legacyToProbe); } + // Each configured shard is read once, in parallel: the groups are disjoint by id, so the + // results need no dedupe. + const shardRows = ( + await Promise.all( + [...idsByShard.entries()].map(([shardKey, ids]) => + input.readNew(shardReplicas.get(shardKey)!, ids) + ) + ) + ).flat(); + // Order within the page is irrelevant (downstream pMap does not depend on it). - return [...newRows, ...legacyRows]; + return [...newRows, ...legacyRows, ...shardRows]; } diff --git a/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts b/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts index eb322c48a1c..ade925f239c 100644 --- a/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts +++ b/apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts @@ -3,10 +3,10 @@ // RESULTS READ assembles correctly when one batch's members are genuinely split across the real // dedicated run-ops subset schema (prisma17 / RunOpsPrismaClient) and the full control-plane // schema (prisma14) — not a mirrored full schema on both sides. No mocks. -import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import { heteroRunOpsPostgresTest, makeNShardRunOpsPostgresTest } from "@internal/testcontainers"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import type { PrismaClient } from "@trigger.dev/database"; -import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { generateRunOpsId, generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; import { describe, expect, vi } from "vitest"; import type { PrismaReplicaClient } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; @@ -209,6 +209,10 @@ async function seedBatchOnNew( return batch; } +// One real gen-2 shard on its OWN database, so a member seeded there is genuinely absent +// from the gen-1 `new` store rather than merely routed away from it. +const oneShardTest = makeNShardRunOpsPostgresTest(1); + const env = (ctx: SeedCtx) => ({ id: ctx.environment.id, @@ -334,4 +338,141 @@ describe("ApiBatchResultsPresenter split mode — real run-ops dedicated schema expect(result!.items[0]).toMatchObject({ ok: true, id: "run_present" }); } ); + + // A gen-2 member is directly routable to its own shard. Before the shard arm existed it + // joined the gen-1 `new` read, missed there, and — classifying dedicated-family — never + // reached the legacy probe either, so it vanished from the batch results with no error. + oneShardTest( + "a gen-2 member is hydrated from its own shard database alongside a legacy-resident member", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const shardPrisma = shardPrismas[0]!; + const ctx = await seedLegacyEnv(legacyPrisma, "gen2-shard"); + await relaxLegacyAttemptFk(legacyPrisma); + await relaxNewBatchItemFk(newPrisma); + + const shardMemberId = generateRunOpsIdV2("a"); + const legacyMemberId = generateLegacyCuid(); + + // The gen-2 member exists ONLY on the shard database. The gen-1 `new` store below is a + // different database, so routing this id there would genuinely miss. + await seedNewMember( + shardPrisma, + { envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id }, + { + id: shardMemberId, + friendlyId: "run_shard_member", + status: "COMPLETED_SUCCESSFULLY", + output: JSON.stringify({ from: "shard-a" }), + } + ); + await seedLegacyMember(legacyPrisma, ctx, { + id: legacyMemberId, + friendlyId: "run_legacy_member", + status: "COMPLETED_WITH_ERRORS", + error: { type: "BUILT_IN_ERROR", name: "Err", message: "boom", stackTrace: "" }, + }); + + const batchFriendlyId = "batch_gen2_shard"; + await seedBatchOnNew(newPrisma, ctx.environment.id, batchFriendlyId, [ + shardMemberId, + legacyMemberId, + ]); + + const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, { + splitEnabled: true, + newClient: newPrisma as unknown as PrismaReplicaClient, + legacyReplica: legacyPrisma as unknown as PrismaReplicaClient, + shardReplicas: new Map([["a", shardPrisma as unknown as PrismaReplicaClient]]), + }); + + const result = await presenter.call(batchFriendlyId, env(ctx)); + + expect(result).toBeDefined(); + expect(result!.items).toHaveLength(2); + const [first, second] = result!.items; + expect(first).toEqual({ + ok: true, + id: "run_shard_member", + taskIdentifier: "my-task", + output: JSON.stringify({ from: "shard-a" }), + outputType: "application/json", + }); + expect(second).toMatchObject({ ok: false, id: "run_legacy_member" }); + }, + 180_000 + ); + + // A gen-2 id naming a shard that is NOT configured must not fall back onto a gen-1 store: + // that reads the wrong database, misses, and (being dedicated-family) never reaches the + // legacy probe, so the member disappears with no error. Drop it, but loudly. + oneShardTest( + "a gen-2 member on an unconfigured shard is dropped without being read from a gen-1 store", + async ({ legacyPrisma, newPrisma, shardPrismas }) => { + const shardPrisma = shardPrismas[0]!; + const ctx = await seedLegacyEnv(legacyPrisma, "gen2-unconfigured"); + await relaxLegacyAttemptFk(legacyPrisma); + await relaxNewBatchItemFk(newPrisma); + + // Shard "z" is not in the configured map; shard "a" is. + const unconfiguredId = generateRunOpsIdV2("z"); + const legacyMemberId = generateLegacyCuid(); + + await seedNewMember( + shardPrisma, + { envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id }, + { id: unconfiguredId, friendlyId: "run_unconfigured", status: "COMPLETED_SUCCESSFULLY" } + ); + await seedLegacyMember(legacyPrisma, ctx, { + id: legacyMemberId, + friendlyId: "run_legacy_member", + status: "COMPLETED_SUCCESSFULLY", + }); + + const batchFriendlyId = "batch_gen2_unconfigured"; + await seedBatchOnNew(newPrisma, ctx.environment.id, batchFriendlyId, [ + unconfiguredId, + legacyMemberId, + ]); + + // A closure-based recorder, not a mock: it records the id sets each store is asked for, + // so the assertion is about real reads rather than about a test double's behaviour. + const askedOf = (label: string, target: RunOpsPrismaClient | PrismaClient) => { + const asked: string[][] = []; + const handle = { + ...target, + taskRun: { + findMany: (args: { where?: { id?: { in?: string[] } } }) => { + asked.push(args.where?.id?.in ?? []); + return (target as unknown as PrismaReplicaClient).taskRun.findMany(args as never); + }, + }, + } as unknown as PrismaReplicaClient; + return { label, asked, handle }; + }; + const genOneNew = askedOf("new", newPrisma); + const legacy = askedOf("legacy", legacyPrisma); + + const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, { + splitEnabled: true, + newClient: genOneNew.handle, + legacyReplica: legacy.handle, + shardReplicas: new Map([["a", shardPrisma as unknown as PrismaReplicaClient]]), + }); + + const result = await presenter.call(batchFriendlyId, env(ctx)); + + // The legacy member still resolves; the unconfigured gen-2 member is dropped. + expect(result).toBeDefined(); + expect(result!.items).toHaveLength(1); + expect(result!.items[0]).toMatchObject({ ok: true, id: "run_legacy_member" }); + + // The unconfigured id was never asked of a gen-1 store. + for (const store of [genOneNew, legacy]) { + for (const ids of store.asked) { + expect(ids).not.toContain(unconfiguredId); + } + } + }, + 180_000 + ); }); diff --git a/apps/webapp/test/readRunForEvent.replicaLag.test.ts b/apps/webapp/test/readRunForEvent.replicaLag.test.ts index 877f920e40f..9817a1b6e55 100644 --- a/apps/webapp/test/readRunForEvent.replicaLag.test.ts +++ b/apps/webapp/test/readRunForEvent.replicaLag.test.ts @@ -195,4 +195,76 @@ describe("readRunForEvent tolerates replica lag on its event-enrichment read", ( expect(onPrimary.friendlyId).toBe("run_rrfe_missing"); } ); + + // (c) SPLIT MODE, the gen-1 run fast path. A cuid run id classifies legacy, and there is no cuid + // run migration, so the new-store probe cannot find it. readRunForEvent declares idKind "run", + // which reads the legacy replica ONLY. The observable difference is the number of reads: one on + // the fast path, two on the old new-then-legacy pair probe. Counted by delegating through the + // real store rather than replacing it. + containerTest( + "readRunForEvent takes ONE read for a cuid run id under split, not a new-then-legacy pair", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_split"); + + const runId = "d".repeat(25); // cuid-shaped -> classifies legacy + const friendlyId = "run_rrfe_split"; + + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_split", + spanId: "span_split", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + }, + }); + + const realStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma as never }); + let findRunCalls = 0; + const countingStore = new Proxy(realStore, { + get(target, prop, receiver) { + if (prop === "findRun") { + return (...args: unknown[]) => { + findRunCalls += 1; + return (target.findRun as (...a: unknown[]) => unknown)(...args); + }; + } + return Reflect.get(target, prop, receiver); + }, + }); + + // The new side MUST miss for the two arms to be distinguishable: a pair probe that finds the + // row on its first read short-circuits and looks identical to the fast path. `missing` makes + // the new-store read return nothing, exactly as it would for a legacy-resident run. + const missingOnNew = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + const deps: EventReadDeps = { + store: countingStore as never, + newReplica: missingOnNew.client as never, + legacyReplica: prisma as never, + splitEnabled: true, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + // The run still resolves — the fast path must not cost the read. + expect(run).not.toBeNull(); + expect(run!.id).toBe(runId); + expect(run!.friendlyId).toBe(friendlyId); + + // ONE read. Two would mean the new store was probed first, which is the arm this removes. + expect(findRunCalls).toBe(1); + } + ); }); diff --git a/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts b/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts index c0b627262f7..09c3327ab57 100644 --- a/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts +++ b/apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts @@ -1,7 +1,7 @@ import { heteroRunOpsPostgresTest, postgresTest } from "@internal/testcontainers"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import type { PrismaClient } from "@trigger.dev/database"; -import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { generateRunOpsId, generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; import { describe, expect, vi } from "vitest"; import type { PrismaReplicaClient } from "~/db.server"; import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; @@ -286,4 +286,105 @@ describe("resolveWaitpointThroughReadThrough (hetero PG14 legacy + dedicated run expect(legacy.calls.length).toBe(0); } ); + + heteroRunOpsPostgresTest( + "gen-2 waitpoint resolves on its OWN shard replica; the gen-1 new store is never read", + async ({ prisma17, prisma14 }) => { + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + const seeded = await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + // The gen-1 new store and the legacy replica are both forbidden: a gen-2 id must + // take one read on its shard and probe nothing else. + const newClient = recording(prisma14, { forbidden: true }); + const legacyReplica = recording(prisma14, { forbidden: true }); + const shardReplica = recording(prisma17); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: newClient.handle, + legacyReplica: legacyReplica.handle, + newPrimary: newClient.handle, + shardReplicas: new Map([["a", shardReplica.handle]]), + }, + }); + + expect(result).not.toBeNull(); + expect(result!.id).toBe(seeded.id); + expect(shardReplica.calls.length).toBe(1); + expect(newClient.calls.length).toBe(0); + expect(legacyReplica.calls.length).toBe(0); + } + ); + + heteroRunOpsPostgresTest( + "a gen-2 waitpoint missing its shard REPLICA falls back to that shard's WRITER, not the gen-1 new writer", + async ({ prisma17, prisma14 }) => { + // Read-your-writes: a token completed immediately after mint may not have replicated. + // The fallback must read the shard's own primary. Reading the gen-1 new writer would + // query the wrong database and return null. + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + const seeded = await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + const shardReplica = recording(prisma14); // lags: does not have the row + const shardWriter = recording(prisma17); // has the row + const forbiddenNewPrimary = recording(prisma17, { forbidden: true }); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: recording(prisma14, { forbidden: true }).handle, + legacyReplica: recording(prisma14, { forbidden: true }).handle, + newPrimary: forbiddenNewPrimary.handle, + shardReplicas: new Map([["a", shardReplica.handle]]), + shardWriters: new Map([["a", shardWriter.handle]]), + }, + }); + + expect(result).not.toBeNull(); + expect(result!.id).toBe(seeded.id); + expect(shardReplica.calls.length).toBe(1); + expect(shardWriter.calls.length).toBe(1); + expect(forbiddenNewPrimary.calls.length).toBe(0); + } + ); + + heteroRunOpsPostgresTest( + "a gen-2 waitpoint with no configured shard writer returns null instead of reading a wrong database", + async ({ prisma17, prisma14 }) => { + const id = generateRunOpsIdV2("a"); + const environmentId = generateRunOpsId(); + const projectId = generateRunOpsId(); + await seedWaitpoint(prisma17, id, { id: environmentId, projectId }); + + const forbiddenNewPrimary = recording(prisma17, { forbidden: true }); + + const result = await resolveWaitpointThroughReadThrough({ + waitpointId: id, + environmentId, + read: read(id, environmentId), + deps: { + splitEnabled: true, + newClient: recording(prisma14, { forbidden: true }).handle, + legacyReplica: recording(prisma14, { forbidden: true }).handle, + newPrimary: forbiddenNewPrimary.handle, + shardReplicas: new Map([["a", recording(prisma14).handle]]), + shardWriters: new Map(), + }, + }); + + expect(result).toBeNull(); + expect(forbiddenNewPrimary.calls.length).toBe(0); + } + ); }); diff --git a/apps/webapp/test/unroutableIdStatus.test.ts b/apps/webapp/test/unroutableIdStatus.test.ts new file mode 100644 index 00000000000..89135b44e15 --- /dev/null +++ b/apps/webapp/test/unroutableIdStatus.test.ts @@ -0,0 +1,40 @@ +// `resolveShard` is pure id-shape, so any base32hex core plus `[a-z0-9]` plus "2" parses as gen-2 +// and names a shard — including one a caller invents. The routing store throws for a key it has no +// store for, which is correct and deliberately loud, but a read route that lets it reach the +// boundary answered 500 for caller-supplied input. These tests pin the boundary status. +import { describe, expect, it } from "vitest"; +import { json } from "@remix-run/server-runtime"; +import { UnknownShardKey } from "@internal/run-store"; +import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server"; + +describe("unroutableIdResponse", () => { + it("answers 404 for an id naming a shard with no configured store", async () => { + const response = unroutableIdResponse(new UnknownShardKey("z", ["legacy", "new"])); + + expect(response).toBeDefined(); + expect(response!.status).toBe(404); + // Not retryable: no number of retries makes a topology grow a store. Contrast the + // waitpoint wait route, whose 404 IS retryable because a miss there can be replica lag. + expect(response!.headers.get("x-should-retry")).toBe("false"); + await expect(response!.json()).resolves.toEqual({ error: "Not Found" }); + }); + + it("declines an unrelated error so it still reaches the 500 path", () => { + expect(unroutableIdResponse(new Error("db down"))).toBeUndefined(); + expect(unroutableIdResponse(undefined)).toBeUndefined(); + expect(unroutableIdResponse("a string")).toBeUndefined(); + }); + + it("declines a deliberately thrown Response, which carries its own status", () => { + expect(unroutableIdResponse(json({ error: "nope" }, { status: 422 }))).toBeUndefined(); + }); + + it("keeps the key and the configured set on the error for the operator", () => { + // A 404 to the caller must not cost the operator what separates a forged id from a shard + // key dropped out of a config that is meant to be append-only. + const error = new UnknownShardKey("z", ["legacy", "new", "a"]); + + expect(error.shardKey).toBe("z"); + expect(error.configured).toContain("a"); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 7af040eb99c..df718b4a1af 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -30,6 +30,7 @@ import type { TaskRunWithWaitpoint, } from "./types.js"; import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; // Loose delegate method shape: each generated client types delegate methods as // `(args: PackageLocalArgs) => PrismaPromise<…>` against its own nominal @@ -2757,7 +2758,7 @@ export class PostgresRunStore implements RunStore { data: { environmentId: string; name: string; projectId: string; id?: string }, tx?: PrismaClientOrTransaction, // `residency` selects the store at the router; a single store has one client and ignores it. - _residency?: "NEW" | "LEGACY" + _residency?: ShardKey ): Promise { const prisma = tx ?? this.prisma; diff --git a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts index b6ea71e9f60..8f2cc8c6485 100644 --- a/internal-packages/run-store/src/runOpsStore.shardMap.test.ts +++ b/internal-packages/run-store/src/runOpsStore.shardMap.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { RoutingRunStore } from "./runOpsStore.js"; +import { RoutingRunStore, UnknownShardKey } from "./runOpsStore.js"; import type { ReadClient, RunStore } from "./types.js"; // Pins the routing ALGEBRA: probe order, merge precedence, and the two id-less fallbacks that @@ -278,6 +278,65 @@ describe("RoutingRunStore id-to-shard-key seam", () => { ); expect(trace(log)).toEqual([]); }); + + // The case above injects a resolver. This one does NOT: it uses the real `resolveShard`, which + // the compat constructor defaults to. `resolveShard` is pure id-shape, so a gen-2 shaped id + // names its shard char whatever the topology holds — the two-store compat router therefore + // reaches this throw for any gen-2 id, with no shard configured anywhere. + // + // That matters beyond this class: these ids reach read routes as URL parameters, so whatever + // sits above the router must translate this throw into a 4xx rather than let it surface as a + // 5xx that any caller can induce. + it("reaches the unconfigured-shard throw for a real gen-2 id, even on the compat pair", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + const genTwoId = `${"0".repeat(24)}a2`; + + expect(() => router.findRun({ id: genTwoId })).toThrow( + 'no store is configured for shard key "a"' + ); + expect(trace(log)).toEqual([]); + }); + + // Typed, not a bare Error: the API boundary matches on it to answer 404 instead of 500, and + // the operator needs the key and the configured set to tell a forged id from a dropped shard. + it("throws a typed UnknownShardKey carrying the key and the configured set", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + let thrown: unknown; + try { + router.findRun({ id: `${"0".repeat(24)}a2` }); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(UnknownShardKey); + const error = thrown as UnknownShardKey; + expect(error.name).toBe("UnknownShardKey"); + expect(error.shardKey).toBe("a"); + expect([...error.configured].sort()).toEqual(["legacy", "new"]); + }); + + it("still routes gen-1 shapes on the compat pair with the real resolver", () => { + const log: Call[] = []; + const router = new RoutingRunStore({ + new: fakeStore("new", log), + legacy: fakeStore("legacy", log), + }); + + router.findRun({ id: `${"0".repeat(24)}01` }); + router.findRun({ id: "c".repeat(25) }); + + expect(trace(log)).toEqual(["new:findRun", "legacy:findRun"]); + }); }); function buildNShardRouter(shardKeys: string[], opts: { aliasOf?: Record } = {}) { @@ -301,6 +360,14 @@ function buildNShardRouter(shardKeys: string[], opts: { aliasOf?: Record { + // findRunsByIds reaches #fanOutPartitioned, the third unconfigured-shard guard. It must throw + // the typed error too, or this read path answers 500 where the boundary would give a 404. + it("throws a typed UnknownShardKey from the partitioned id fan-out", async () => { + const { router } = buildNShardRouter(["a"]); + + await expect(router.findRunsByIds(["a:r1", "z:r2"])).rejects.toBeInstanceOf(UnknownShardKey); + }); + it("routes an id to its gen-2 shard", async () => { const { router, log } = buildNShardRouter(["a", "b"]); await router.findRun({ id: "a:run_1" }); @@ -649,6 +716,17 @@ describe("RoutingRunStore countPendingWaitpoints — disjoint-sum partition", () ); }); + // The API boundary answers a non-retryable 404 by matching on the TYPE, so every + // unconfigured-shard guard has to throw the typed error and not a bare Error. Two other guards + // besides #shardStore reach an unconfigured key: this partition, and #fanOutPartitioned below. + it("throws a typed UnknownShardKey from the absent-id partition", async () => { + const { router } = partitionRouter({}); + + await expect( + router.countPendingWaitpoints(["c:w1"], undefined, "a:run") + ).rejects.toBeInstanceOf(UnknownShardKey); + }); + it("returns zero for an id absent everywhere", async () => { const { router } = partitionRouter({}); expect(await router.countPendingWaitpoints(["b:w9"], undefined, "a:run")).toBe(0); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 53089da21a5..7551e552b34 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -62,6 +62,28 @@ const LEGACY_SHARD: ShardKey = "legacy"; * and a merge lets a gen-2 shard win. A probe MUST iterate #probeOrder and a merge MUST iterate * #precedence. */ +/** + * An id resolved to a shard key the topology has no store for. Typed so a caller above the + * router can answer a 4xx instead of letting a routing failure surface as a 5xx: these ids + * arrive as URL parameters, and `resolveShard` is pure id-shape, so any gen-2 shaped id names + * a shard char whether or not one is configured. + */ +export class UnknownShardKey extends Error { + readonly shardKey: string; + readonly configured: string[]; + + constructor(shardKey: string, configured: string[], subject?: string) { + super( + subject === undefined + ? `RoutingRunStore: no store is configured for shard key "${shardKey}"` + : `RoutingRunStore: ${subject} resolves to unconfigured shard key "${shardKey}"` + ); + this.name = "UnknownShardKey"; + this.shardKey = shardKey; + this.configured = configured; + } +} + export class RoutingRunStore implements RunStore { readonly #shards: ReadonlyMap; // Sequential probe for a lookup with no routable id. The first non-null result wins, and the LAST @@ -173,12 +195,14 @@ export class RoutingRunStore implements RunStore { return client != null && !isReadReplicaClient(client) ? store.primaryReadClient : undefined; } - // The store for a shard key. Unreachable with the compat constructor — #shardKeyOfSafe yields only - // the two reserved keys — so this throw fires only if a caller wires a partial map. + // The store for a shard key. REACHABLE with the compat constructor: it defaults to the real + // `resolveShard`, which is pure id-shape, so any gen-2 shaped id names a shard char even when + // no shard is configured. Fails loud rather than reading the wrong database; the API boundary + // turns `UnknownShardKey` into a 404 so a caller-supplied id cannot induce a 5xx. #shardStore(key: ShardKey): RunStore { const store = this.#shards.get(key); if (store === undefined) { - throw new Error(`RoutingRunStore: no store is configured for shard key "${key}"`); + throw new UnknownShardKey(key, [...this.#shards.keys()]); } return store; } @@ -236,9 +260,7 @@ export class RoutingRunStore implements RunStore { // Fail loud instead (§7 append-only rule). if (key === runKey) return; if (!this.#shards.has(key)) { - throw new Error( - `RoutingRunStore: waitpoint "${id}" resolves to unconfigured shard key "${key}"` - ); + throw new UnknownShardKey(key, [...this.#shards.keys()], `waitpoint "${id}"`); } const bucket = byKey.get(key); if (bucket) bucket.push(id); @@ -393,7 +415,7 @@ export class RoutingRunStore implements RunStore { // An id resolving to a shard nobody configured is UnknownShardKey. Dropping it would silently // omit a row from the hydrated set, so fail loud (§7 append-only rule). if (!this.#shards.has(key)) { - throw new Error(`RoutingRunStore: id "${id}" resolves to unconfigured shard key "${key}"`); + throw new UnknownShardKey(key, [...this.#shards.keys()], `id "${id}"`); } const bucket = byShard.get(key); if (bucket) bucket.push(id); From c7f78e4853b0a86584f17b1924626b85f279fda1 Mon Sep 17 00:00:00 2001 From: wei-wei Date: Wed, 26 Aug 2026 10:56:32 -0700 Subject: [PATCH 25/28] fix(sdk): reset skipToTurnComplete when a new chat turn starts (#4744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing Reproduced with `useTriggerChatTransport` + `useChat` and the stop pattern from the ai-chat frontend docs: 1. Send a message so a turn is streaming. 2. Call `transport.stopGeneration(chatId)`, then `useChat`'s `stop()`. 3. Send another message. Before this change the second turn never renders: no parts arrive, `status` stays `streaming`, and the session stays `isStreaming: true`, so a stop button stays on screen until the page is reloaded. The run itself is fine and everything persists, so a reload shows the full response. Cause: `stopGeneration` sets `state.skipToTurnComplete = true`, and the read loop only clears that when it sees a `TURN_COMPLETE` record. The abort closes the reader before that record arrives, so the flag survives into the next turn and every record of that turn is skipped, including its own `TURN_COMPLETE`. After this change the same sequence streams the second turn normally. Verified against 4.5.11 and 4.5.12 (both affected) with the equivalent patch applied to the built SDK. --- ## Changelog Reset `skipToTurnComplete` when a new chat turn or action is sent, so a message sent after `stopGeneration` streams normally instead of leaving the chat stuck in a streaming state. --------- Co-authored-by: Devin AI Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com> --- .changeset/fluffy-pans-argue.md | 5 ++ packages/trigger-sdk/src/v3/chat.ts | 8 +++ .../test/chat-transport-events.test.ts | 64 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 .changeset/fluffy-pans-argue.md diff --git a/.changeset/fluffy-pans-argue.md b/.changeset/fluffy-pans-argue.md new file mode 100644 index 00000000000..5bbe81a0fa7 --- /dev/null +++ b/.changeset/fluffy-pans-argue.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Fix chat transport discarding the next turn after stopping generation. `skipToTurnComplete` is now reset when a new message or action is sent, so a message sent after `stopGeneration` streams normally instead of leaving the chat stuck in a streaming state. diff --git a/packages/trigger-sdk/src/v3/chat.ts b/packages/trigger-sdk/src/v3/chat.ts index a7c7125575e..55889bd58e5 100644 --- a/packages/trigger-sdk/src/v3/chat.ts +++ b/packages/trigger-sdk/src/v3/chat.ts @@ -870,6 +870,10 @@ export class TriggerChatTransport implements ChatTransport { this.activeStreams.delete(chatId); } + // A stop that never saw its TURN_COMPLETE leaves the flag set, and the new + // turn would be skipped record by record. + state.skipToTurnComplete = false; + state.isStreaming = true; this.notifySessionChange(chatId, state); @@ -1281,6 +1285,10 @@ export class TriggerChatTransport implements ChatTransport { this.activeStreams.delete(chatId); } + // A stop that never saw its TURN_COMPLETE leaves the flag set, and the new + // turn would be skipped record by record. + state.skipToTurnComplete = false; + // Mark streaming + persist so a reload mid-action resumes (reconnectToStream // no-ops when the persisted session says isStreaming: false). state.isStreaming = true; diff --git a/packages/trigger-sdk/test/chat-transport-events.test.ts b/packages/trigger-sdk/test/chat-transport-events.test.ts index 39f4e53d722..a35e78ad519 100644 --- a/packages/trigger-sdk/test/chat-transport-events.test.ts +++ b/packages/trigger-sdk/test/chat-transport-events.test.ts @@ -174,6 +174,70 @@ describe("transport send events", () => { }); }); +describe("stopped turn followed by a new turn", () => { + /** + * `.out` stub that honours the `Last-Event-ID` cursor like the server does, so + * a resubscribe cannot replay records the reader already consumed. A stop that + * never saw its turn-complete is therefore unrecoverable unless the new send + * clears the skip state. + */ + function cursoredOneTurnTransport() { + const frames = [ + { id: "1", data: `{"type":"text-delta","id":"t1","delta":"hello"}` }, + { id: "2", data: `{"type":"trigger:turn-complete"}` }, + ]; + + return makeTransport({ + fetch: async (_url, init, ctx) => { + if (ctx.endpoint === "in") return jsonOk(); + + const cursor = new Headers(init.headers).get("Last-Event-ID"); + const from = cursor ? frames.findIndex((f) => f.id === cursor) + 1 : 0; + const remaining = frames.slice(from); + const response = sseResponse( + remaining.map((f) => `id: ${f.id}\ndata: ${f.data}\n\n`).join("") + ); + // Nothing left to send: the session is settled, so the reader stops + // instead of resubscribing. + if (remaining.length === 0) response.headers.set("X-Session-Settled", "true"); + return response; + }, + }); + } + + it("streams a sendMessages turn after a stop that never saw turn-complete", async () => { + const { transport, events } = cursoredOneTurnTransport(); + + expect(await transport.stopGeneration("c1")).toBe(true); + events.length = 0; + + const stream = await transport.sendMessages({ + trigger: "submit-message", + chatId: "c1", + messageId: undefined, + messages: [user("after stop", "u-2")], + abortSignal: undefined, + }); + const chunks = await readAll(stream); + + expect(chunks).toEqual([{ type: "text-delta", id: "t1", delta: "hello" }]); + expect(events.some((e) => e.type === "turn-completed")).toBe(true); + }); + + it("streams a sendAction turn after a stop that never saw turn-complete", async () => { + const { transport, events } = cursoredOneTurnTransport(); + + expect(await transport.stopGeneration("c1")).toBe(true); + events.length = 0; + + const stream = await transport.sendAction("c1", { type: "undo" }); + const chunks = await readAll(stream); + + expect(chunks).toEqual([{ type: "text-delta", id: "t1", delta: "hello" }]); + expect(events.some((e) => e.type === "turn-completed")).toBe(true); + }); +}); + describe("transport stream events", () => { it("marks reconnectToStream subscriptions as resumed", async () => { const { transport, events } = makeTransport({ From b72c6c3a88077766894d8255620d57eedde49c36 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Thu, 27 Aug 2026 06:34:37 +0000 Subject: [PATCH 26/28] feat(sdk): allow a ttl on session-triggered runs --- .changeset/chat-session-run-ttl.md | 6 ++++++ packages/core/src/v3/schemas/api.ts | 5 +++++ packages/trigger-sdk/src/v3/ai.ts | 2 ++ packages/trigger-sdk/src/v3/chat-server.test.ts | 4 +++- packages/trigger-sdk/src/v3/chat-server.ts | 1 + .../src/v3/createStartSessionAction.test.ts | 13 ++++++++++++- 6 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .changeset/chat-session-run-ttl.md diff --git a/.changeset/chat-session-run-ttl.md b/.changeset/chat-session-run-ttl.md new file mode 100644 index 00000000000..8169b800631 --- /dev/null +++ b/.changeset/chat-session-run-ttl.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Chat server sessions can now set a `ttl` on the runs they trigger, so a run that is never picked up expires instead of waiting indefinitely. diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index a90430953d4..14b6f54ea79 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -1850,6 +1850,11 @@ export const SessionTriggerConfig = z.object({ lockToVersion: z.string().optional(), /** Region to schedule runs in. Forwarded to `TaskRunOptions.region`. */ region: z.string().optional(), + /** + * How long a run may sit undequeued before it expires (duration string + * like `"2m"`, or seconds). Forwarded to `TaskRunOptions.ttl`. + */ + ttl: z.string().or(z.number().nonnegative().int()).optional(), /** Convenience field surfaced to chat.agent via the wire payload. */ idleTimeoutInSeconds: z.number().int().positive().max(3600).optional(), }); diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index c241930323b..dfee30f039a 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -10448,6 +10448,7 @@ function createChatStartSessionAction( const maxDuration = params.triggerConfig?.maxDuration ?? options?.triggerConfig?.maxDuration; const idleTimeoutInSeconds = params.triggerConfig?.idleTimeoutInSeconds ?? options?.triggerConfig?.idleTimeoutInSeconds; + const ttl = params.triggerConfig?.ttl ?? options?.triggerConfig?.ttl; const triggerConfig: SessionTriggerConfig = { basePayload: { @@ -10470,6 +10471,7 @@ function createChatStartSessionAction( ...(options?.triggerConfig?.region || params.triggerConfig?.region ? { region: params.triggerConfig?.region ?? options?.triggerConfig?.region } : {}), + ...(ttl !== undefined ? { ttl } : {}), ...(options?.triggerConfig?.lockToVersion || params.triggerConfig?.lockToVersion ? { lockToVersion: diff --git a/packages/trigger-sdk/src/v3/chat-server.test.ts b/packages/trigger-sdk/src/v3/chat-server.test.ts index 539fe0247ad..92ffd77502c 100644 --- a/packages/trigger-sdk/src/v3/chat-server.test.ts +++ b/packages/trigger-sdk/src/v3/chat-server.test.ts @@ -216,7 +216,7 @@ describe("chat.headStart (route handler)", () => { expect(body.triggerConfig.basePayload.idleTimeoutInSeconds).toBe(60); }); - it("merges triggerConfig tags and queue into createSession", async () => { + it("merges triggerConfig tags, queue and ttl into createSession", async () => { const requests: CapturedRequest[] = []; global.fetch = vi.fn().mockImplementation(async (url: string | URL, init?: RequestInit) => { const urlStr = typeof url === "string" ? url : url.toString(); @@ -248,6 +248,7 @@ describe("chat.headStart (route handler)", () => { triggerConfig: { tags: ["org:acme", "agentic-run:xyz"], queue: "my-queue", + ttl: "2m", }, run: async ({ chat: chatHelper }) => { return streamText({ @@ -276,6 +277,7 @@ describe("chat.headStart (route handler)", () => { const body = JSON.parse(sessionCreate!.init!.body as string); expect(body.triggerConfig.tags).toEqual(["chat:chat-1", "org:acme", "agentic-run:xyz"]); expect(body.triggerConfig.queue).toBe("my-queue"); + expect(body.triggerConfig.ttl).toBe("2m"); expect(body.triggerConfig.basePayload.trigger).toBe("handover-prepare"); expect(body.triggerConfig.basePayload.chatId).toBe("chat-1"); }); diff --git a/packages/trigger-sdk/src/v3/chat-server.ts b/packages/trigger-sdk/src/v3/chat-server.ts index 5e48e3b24b6..dc850d118ed 100644 --- a/packages/trigger-sdk/src/v3/chat-server.ts +++ b/packages/trigger-sdk/src/v3/chat-server.ts @@ -550,6 +550,7 @@ async function openHandoverSession(opts: { ? { maxDuration: opts.triggerConfig.maxDuration } : {}), ...(opts.triggerConfig?.region ? { region: opts.triggerConfig.region } : {}), + ...(opts.triggerConfig?.ttl !== undefined ? { ttl: opts.triggerConfig.ttl } : {}), ...(opts.triggerConfig?.lockToVersion ? { lockToVersion: opts.triggerConfig.lockToVersion } : {}), diff --git a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts index ca18ce59985..ca51282e614 100644 --- a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts +++ b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts @@ -115,7 +115,7 @@ describe("chat.createStartSessionAction — runtime", () => { ]); }); - it("forwards maxDuration, region, and lockToVersion from triggerConfig", async () => { + it("forwards maxDuration, region, lockToVersion, and ttl from triggerConfig", async () => { installStartFixture(); const start = chat.createStartSessionAction("fake-chat", { @@ -123,6 +123,7 @@ describe("chat.createStartSessionAction — runtime", () => { maxDuration: 120, region: "us-east-1", lockToVersion: "20260101.1", + ttl: "2m", }, }); await start({ chatId: "chat-parity" }); @@ -130,6 +131,16 @@ describe("chat.createStartSessionAction — runtime", () => { expect(lastStartBody?.triggerConfig.maxDuration).toBe(120); expect(lastStartBody?.triggerConfig.region).toBe("us-east-1"); expect(lastStartBody?.triggerConfig.lockToVersion).toBe("20260101.1"); + expect(lastStartBody?.triggerConfig.ttl).toBe("2m"); + }); + + it("omits ttl when triggerConfig does not set it", async () => { + installStartFixture(); + + const start = chat.createStartSessionAction("fake-chat"); + await start({ chatId: "chat-no-ttl" }); + + expect(lastStartBody?.triggerConfig).not.toHaveProperty("ttl"); }); it("server-mints override tokens for additional API keys", async () => { From 432f17bb31d972e8a1061bcb4f267160a9535822 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Thu, 27 Aug 2026 06:34:38 +0000 Subject: [PATCH 27/28] fix(webapp): expire dashboard agent turn runs that are never dequeued --- apps/webapp/app/services/dashboardAgent.server.ts | 12 ++++++++++-- .../services/realtime/sessionRunManager.server.ts | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 5ff460ae33f..7252ba2d178 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -86,10 +86,18 @@ export function isDashboardAgentConfigured(): boolean { return Boolean(env.DASHBOARD_AGENT_SECRET_KEY); } +// With no agent worker available a turn's run would sit queued indefinitely and +// could be dequeued much later with a stale token. Expire it instead — the turn +// is long dead by then on the client. +const DASHBOARD_AGENT_RUN_TTL = "2m"; + // Pins every agent session (and its continuation runs) to a deployed version // when DASHBOARD_AGENT_VERSION is set; unset runs on the env's current version. -export function dashboardAgentTriggerConfig(): { lockToVersion: string } | undefined { - return env.DASHBOARD_AGENT_VERSION ? { lockToVersion: env.DASHBOARD_AGENT_VERSION } : undefined; +export function dashboardAgentTriggerConfig(): { ttl: string; lockToVersion?: string } { + return { + ttl: DASHBOARD_AGENT_RUN_TTL, + ...(env.DASHBOARD_AGENT_VERSION ? { lockToVersion: env.DASHBOARD_AGENT_VERSION } : {}), + }; } export async function startDashboardAgentSession(params: { diff --git a/apps/webapp/app/services/realtime/sessionRunManager.server.ts b/apps/webapp/app/services/realtime/sessionRunManager.server.ts index a1989a9ef7a..f11bc960205 100644 --- a/apps/webapp/app/services/realtime/sessionRunManager.server.ts +++ b/apps/webapp/app/services/realtime/sessionRunManager.server.ts @@ -310,6 +310,7 @@ async function triggerSessionRun(params: { ...(config.maxDuration !== undefined ? { maxDuration: config.maxDuration } : {}), ...(config.lockToVersion ? { lockToVersion: config.lockToVersion } : {}), ...(config.region ? { region: config.region } : {}), + ...(config.ttl !== undefined ? { ttl: config.ttl } : {}), }, }; From 80b3fd3e7881a66ba3b4fb8729b8ca835aa7546d Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Thu, 27 Aug 2026 06:40:19 +0000 Subject: [PATCH 28/28] test(webapp): assert the session ttl reaches the trigger options --- apps/webapp/test/realtimeServices.replicaLag.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/webapp/test/realtimeServices.replicaLag.test.ts b/apps/webapp/test/realtimeServices.replicaLag.test.ts index 6a302dcfd9c..0de58c0403d 100644 --- a/apps/webapp/test/realtimeServices.replicaLag.test.ts +++ b/apps/webapp/test/realtimeServices.replicaLag.test.ts @@ -395,7 +395,7 @@ describe("realtime-svc — replica-lag guards", () => { environmentType: "DEVELOPMENT", organizationId: seed.organization.id, taskIdentifier: "my-task", - triggerConfig: { basePayload: {} }, + triggerConfig: { basePayload: {}, ttl: "2m" }, currentRunId: callingRunId, currentRunVersion: 0, streamBasinName: "session-pinned-basin", @@ -429,6 +429,8 @@ describe("realtime-svc — replica-lag guards", () => { // previousRunId forwarded to the triggered run is the calling run's cuid (documented fallback). expect(triggerState.calls).toHaveLength(1); expect(triggerState.calls[0]!.body.payload.previousRunId).toBe(callingRunId); + // The session's ttl reaches the trigger options, so an undequeued run expires. + expect(triggerState.calls[0]!.body.options.ttl).toBe("2m"); expect(versionCalls.at(-1)).toEqual({ requested: "v2", basin: null }); expect(replica.wasHit("taskRun")).toBe(true);